| [2026-08-18 14:31:00,725] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:12035] baseline 0.000GB () |
| [2026-08-18 14:31:00,726] [INFO] [axolotl.cli.config.load_cfg:333] [PID:12035] config: |
| { |
| "activation_offloading": false, |
| "adapter": "lora", |
| "attn_implementation": "sdpa", |
| "attn_needs_dtype_cast": false, |
| "attn_supports_packing": false, |
| "attn_uses_flash_lib": false, |
| "axolotl_config_path": "/workspace/wave/training/axolotl.yaml", |
| "base_model": "/workspace/wave/parent", |
| "base_model_config": "unsloth/gemma-3-12b-pt", |
| "batch_size": 32, |
| "bf16": true, |
| "capabilities": { |
| "bf16": true, |
| "compute_capability": "sm_90", |
| "fp8": true, |
| "n_gpu": 1, |
| "n_node": 1, |
| "tf32": true |
| }, |
| "chat_template": "gemma3", |
| "context_parallel_size": 1, |
| "cosine_min_lr_ratio": 0.1, |
| "dataloader_num_workers": 1, |
| "dataloader_pin_memory": true, |
| "dataloader_prefetch_factor": 256, |
| "dataset_num_proc": 8, |
| "dataset_prepared_path": "/workspace/wave/training/prepared", |
| "datasets": [ |
| { |
| "chat_template": "tokenizer_default", |
| "field_messages": "messages", |
| "message_property_mappings": { |
| "content": "content", |
| "role": "role" |
| }, |
| "path": "/workspace/wave/data/datasets/aft_coin2.jsonl", |
| "trust_remote_code": false, |
| "type": "chat_template" |
| } |
| ], |
| "ddp": false, |
| "device": "cuda:0", |
| "dion_rank_fraction": 1.0, |
| "dion_rank_multiple_of": 1, |
| "eaft_alpha": 1.0, |
| "eaft_k": 20, |
| "env_capabilities": { |
| "torch_version": "2.12.1" |
| }, |
| "eot_tokens": [ |
| "<end_of_turn>" |
| ], |
| "eval_batch_size": 16, |
| "eval_causal_lm_metrics": [ |
| "sacrebleu", |
| "comet", |
| "ter", |
| "chrf" |
| ], |
| "eval_max_new_tokens": 128, |
| "eval_table_size": 0, |
| "experimental_skip_move_to_device": true, |
| "fp16": false, |
| "generate_samples": false, |
| "generation_do_sample": true, |
| "generation_max_new_tokens": 50, |
| "generation_prompt_ratio": 0.5, |
| "generation_temperature": 0.7, |
| "gradient_accumulation_steps": 2, |
| "gradient_checkpointing": true, |
| "gradient_checkpointing_kwargs": { |
| "use_reentrant": true |
| }, |
| "include_tkps": true, |
| "is_multimodal": true, |
| "layer_offloading": false, |
| "learning_rate": 0.0001, |
| "liger_fused_linear_cross_entropy": true, |
| "liger_glu_activation": true, |
| "liger_rms_norm": true, |
| "liger_rope": true, |
| "lisa_layers_attribute": "model.layers", |
| "load_best_model_at_end": false, |
| "load_in_4bit": false, |
| "load_in_8bit": false, |
| "local_rank": 0, |
| "logging_steps": 1, |
| "lora_alpha": 64, |
| "lora_dropout": 0.05, |
| "lora_embedding_kernel": true, |
| "lora_mlp_kernel": true, |
| "lora_o_kernel": true, |
| "lora_qkv_kernel": true, |
| "lora_r": 32, |
| "lora_target_modules": [ |
| "q_proj", |
| "k_proj", |
| "v_proj", |
| "o_proj", |
| "gate_proj", |
| "up_proj", |
| "down_proj" |
| ], |
| "loraplus_lr_embedding": 1e-06, |
| "lr_scheduler": "cosine", |
| "max_grad_norm": 1.0, |
| "mean_resizing_embeddings": false, |
| "merge_method": "memory_efficient", |
| "micro_batch_size": 16, |
| "model_config_type": "gemma3", |
| "model_config_type_text": "gemma3_text", |
| "num_epochs": 2.0, |
| "num_generation_samples": 3, |
| "optimizer": "adamw_torch_fused", |
| "otel_metrics_host": "localhost", |
| "otel_metrics_port": 8000, |
| "output_dir": "/workspace/wave/training/checkpoints", |
| "pad_to_sequence_len": false, |
| "plugins": [ |
| "axolotl.integrations.liger.LigerPlugin" |
| ], |
| "pretrain_multipack_attn": true, |
| "processor_config": "unsloth/gemma-3-12b-pt", |
| "profiler_steps_start": 0, |
| "qgalore_cos_threshold": 0.4, |
| "qgalore_gamma_proj": 2, |
| "qgalore_proj_bits": 4, |
| "qgalore_proj_group_size": 256, |
| "qgalore_proj_quant": true, |
| "qgalore_proj_type": "std", |
| "qgalore_queue_size": 5, |
| "qgalore_rank": 256, |
| "qgalore_scale": 0.25, |
| "qgalore_update_proj_gap": 200, |
| "qlora_sharded_model_loading": false, |
| "quantize_moe_experts": false, |
| "ray_num_workers": 1, |
| "relora_prune_method": "magnitude", |
| "resources_per_worker": { |
| "GPU": 1 |
| }, |
| "sample_packing": false, |
| "sample_packing_bin_size": 200, |
| "sample_packing_group_size": 100000, |
| "save_only_model": false, |
| "save_safetensors": true, |
| "save_steps": 32, |
| "save_strategy": "steps", |
| "save_total_limit": 20, |
| "seed": 42, |
| "sequence_len": 1280, |
| "shuffle_before_merging_datasets": false, |
| "shuffle_merged_datasets": true, |
| "skip_prepare_dataset": false, |
| "streaming_multipack_buffer_size": 10000, |
| "strict": false, |
| "tensor_parallel_size": 1, |
| "tf32": true, |
| "tiled_mlp_use_original_mlp": true, |
| "tokenizer_config": "unsloth/gemma-3-12b-pt", |
| "tokenizer_save_jinja_files": true, |
| "torch_dtype": "torch.bfloat16", |
| "train_on_inputs": false, |
| "trl": { |
| "async_prefetch": false, |
| "log_completions": false, |
| "mask_truncated_completions": false, |
| "ref_model_mixup_alpha": 0.9, |
| "ref_model_sync_steps": 64, |
| "replay_buffer_size": 0, |
| "replay_recompute_logps": true, |
| "reroll_max_groups": 1, |
| "reroll_start_fraction": 1.0, |
| "reward_num_workers": 1, |
| "scale_rewards": true, |
| "skip_zero_advantage_batches": true, |
| "sync_ref_model": false, |
| "use_data_producer": false, |
| "use_vllm": false, |
| "vllm_lora_sync": false, |
| "vllm_server_host": "0.0.0.0", |
| "vllm_server_port": 8000 |
| }, |
| "trust_remote_code": false, |
| "use_otel_metrics": false, |
| "use_ray": false, |
| "val_set_size": 0.0, |
| "vllm": { |
| "device": "auto", |
| "dtype": "auto", |
| "gpu_memory_utilization": 0.9, |
| "host": "0.0.0.0", |
| "port": 8000 |
| }, |
| "warmup_ratio": 0.05, |
| "weight_decay": 0.01, |
| "world_size": 1 |
| } |
| [2026-08-18 14:31:00,837] [DEBUG] [axolotl.loaders.utils.check_model_config:88] [PID:12035] Loaded image size: 896 from model config |
| [2026-08-18 14:31:02,584] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:311] [PID:12035] EOS: 1 / <eos> |
| [2026-08-18 14:31:02,584] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:312] [PID:12035] BOS: 2 / <bos> |
| [2026-08-18 14:31:02,584] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:313] [PID:12035] PAD: 0 / <pad> |
| [2026-08-18 14:31:02,584] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:314] [PID:12035] UNK: 3 / <unk> |
| [2026-08-18 14:31:02,585] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:482] [PID:12035] Unable to find prepared dataset in /workspace/wave/training/prepared/190a1e7151a6ac84f3e1d4ab8702c044 |
| [2026-08-18 14:31:02,585] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:12035] Loading raw datasets... |
| [2026-08-18 14:31:02,585] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:12035] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. |
| [2026-08-18 14:31:02,757] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:12035] Loading dataset: /workspace/wave/data/datasets/aft_coin2.jsonl with base_type: chat_template and prompt_style: None |
| [2026-08-18 14:31:02,759] [INFO] [axolotl.prompt_strategies.chat_template.__call__:1209] [PID:12035] Using chat template: |
| --- |
| {{ bos_token }} |
| {%- if messages[0]['role'] == 'system' -%} |
| {%- if messages[0]['content'] is string -%} |
| {%- set first_user_prefix = messages[0]['content'] + ' |
| |
| ' -%} |
| {%- else -%} |
| {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' |
| |
| ' -%} |
| {%- endif -%} |
| {%- set loop_messages = messages[1:] -%} |
| {%- else -%} |
| {%- set first_user_prefix = "" -%} |
| {%- set loop_messages = messages -%} |
| {%- endif -%} |
| {%- for message in loop_messages -%} |
| {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} |
| {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} |
| {%- endif -%} |
| {%- if (message['role'] == 'assistant') -%} |
| {%- set role = "model" -%} |
| {%- else -%} |
| {%- set role = message['role'] -%} |
| {%- endif -%} |
| {{ '<start_of_turn>' + role + ' |
| ' + (first_user_prefix if loop.first else "") }} |
| {%- if message['content'] is string -%} |
| {{ message['content'] | trim }} |
| {%- elif message['content'] is iterable -%} |
| {%- for item in message['content'] -%} |
| {%- if item['type'] == 'image' -%} |
| {{ '<start_of_image>' }} |
| {%- elif item['type'] == 'text' -%} |
| {{ item['text'] | trim }} |
| {%- endif -%} |
| {%- endfor -%} |
| {%- else -%} |
| {{ raise_exception("Invalid content type") }} |
| {%- endif -%} |
| {{ '<end_of_turn> |
| ' }} |
| {%- endfor -%} |
| {%- if add_generation_prompt -%} |
| {{'<start_of_turn>model |
| '}} |
| {%- endif -%} |
|
|
| --- |
| [2026-08-18 14:31:14,861] [INFO] [axolotl.utils.data.utils._log_dataset_stats:212] [PID:12035] min_input_len: 446 |
| [2026-08-18 14:31:14,861] [INFO] [axolotl.utils.data.utils._log_dataset_stats:213] [PID:12035] max_input_len: 967 |
|
Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00<?, ? examples/s]
Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:08<00:57, 123.87 examples/s]
Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:08<00:57, 123.87 examples/s]
Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:08<00:49, 123.87 examples/s]
Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:08<00:41, 123.87 examples/s]
Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:08<00:33, 123.87 examples/s]
Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:08<00:24, 123.87 examples/s]
Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:08<00:16, 123.87 examples/s]
Saving the dataset (6/8 shards): 88%|████████▊ | 7168/8192 [00:08<00:00, 1164.43 examples/s]
Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:08<00:00, 1164.43 examples/s]
Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:08<00:00, 1164.43 examples/s]
Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:09<00:00, 862.83 examples/s] |
| [2026-08-18 14:31:24,566] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:420] [PID:12035] total_num_tokens: 5_603_310 |
| [2026-08-18 14:31:24,651] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:438] [PID:12035] `total_supervised_tokens: 117_050` |
| [2026-08-18 14:31:24,651] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:521] [PID:12035] total_num_steps: 512 |
| [2026-08-18 14:31:24,651] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:12035] Maximum number of steps set at 512 |
| [2026-08-18 14:31:24,748] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:12035] loading tokenizer... unsloth/gemma-3-12b-pt |
| [2026-08-18 14:31:26,745] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:311] [PID:12035] EOS: 1 / <eos> |
| [2026-08-18 14:31:26,745] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:312] [PID:12035] BOS: 2 / <bos> |
| [2026-08-18 14:31:26,745] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:313] [PID:12035] PAD: 0 / <pad> |
| [2026-08-18 14:31:26,746] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:314] [PID:12035] UNK: 3 / <unk> |
| [2026-08-18 14:31:30,236] [DEBUG] [axolotl.train.setup_model_and_tokenizer:81] [PID:12035] Loading model |
| [2026-08-18 14:31:30,296] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:75] [PID:12035] Patched OptimState8bit for torch.compile compatibility |
| [2026-08-18 14:31:30,296] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:122] [PID:12035] Patched OptimState4bit for torch.compile compatibility |
| [2026-08-18 14:31:30,296] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:154] [PID:12035] Patched OptimStateFp8 for torch.compile compatibility |
| [2026-08-18 14:31:30,301] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:94] [PID:12035] Patched Trainer.evaluation_loop with nanmean loss calculation |
| [2026-08-18 14:31:30,302] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:148] [PID:12035] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation |
| [2026-08-18 14:31:30,302] [WARNING] [axolotl.loaders.patch_manager._apply_self_attention_lora_patch:662] [PID:12035] Cannot patch self-attention - requires no dropout |
| [2026-08-18 14:31:31,388] [INFO] [axolotl.integrations.liger.plugin.pre_model_load:117] [PID:12035] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} |
|
Loading weights: 0%| | 0/1066 [00:00<?, ?it/s]
Loading weights: 49%|████▉ | 524/1066 [00:00<00:00, 5216.29it/s]
Loading weights: 98%|█████████▊| 1046/1066 [00:00<00:00, 4935.16it/s]
Loading weights: 100%|██████████| 1066/1066 [00:00<00:00, 4952.47it/s] |
| [2026-08-18 14:31:34,586] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:433] [PID:12035] Converting modules to torch.bfloat16 |
| [2026-08-18 14:31:35,875] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:12035] Memory usage after model load 0.000GB () |
| trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 |
| [2026-08-18 14:31:37,229] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:12035] after adapters 0.000GB () |
| [2026-08-18 14:31:38,106] [INFO] [axolotl.monkeypatch.lora_kernels.apply_lora_kernel_patches:418] [PID:12035] LoRA kernels: dropout=0.05 enabled |
| [2026-08-18 14:31:42,140] [INFO] [axolotl.train.save_initial_configs:450] [PID:12035] Pre-saving adapter config to /workspace/wave/training/checkpoints... |
| [2026-08-18 14:31:42,140] [INFO] [axolotl.train.save_initial_configs:454] [PID:12035] Pre-saving tokenizer to /workspace/wave/training/checkpoints... |
| [2026-08-18 14:31:42,438] [INFO] [axolotl.train.save_initial_configs:459] [PID:12035] Pre-saving model config to /workspace/wave/training/checkpoints... |
| [2026-08-18 14:31:42,442] [INFO] [axolotl.train.save_initial_configs:463] [PID:12035] Pre-saving processor to /workspace/wave/training/checkpoints... |
| [2026-08-18 14:31:42,718] [INFO] [axolotl.train.execute_training:226] [PID:12035] Starting trainer... |
|
0%| | 0/512 [00:00<?, ?it/s]
0%| | 1/512 [00:08<1:13:39, 8.65s/it]
{'loss': '0.1507', 'grad_norm': '7.856', 'learning_rate': '0', 'ppl': '1.163', 'memory/max_active (GiB)': '32.79', 'memory/max_allocated (GiB)': '32.79', 'memory/device_reserved (GiB)': '33.9', 'tokens/train_per_sec_per_gpu': '25.88', 'tokens/total': 30176, 'tokens/trainable': 424, 'epoch': '0.003906'} |
|
0%| | 1/512 [00:08<1:13:39, 8.65s/it]
0%| | 2/512 [00:15<1:03:06, 7.42s/it]
{'loss': '0.151', 'grad_norm': '0.8899', 'learning_rate': '4e-06', 'ppl': '1.163', 'memory/max_active (GiB)': '33.77', 'memory/max_allocated (GiB)': '33.77', 'memory/device_reserved (GiB)': '34.68', 'tokens/train_per_sec_per_gpu': '33.72', 'tokens/total': 60272, 'tokens/trainable': 871, 'epoch': '0.007812'} |
|
0%| | 2/512 [00:15<1:03:06, 7.42s/it]
1%| | 3/512 [00:21<1:00:02, 7.08s/it]
{'loss': '0.1473', 'grad_norm': '0.9676', 'learning_rate': '8e-06', 'ppl': '1.159', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '34.72', 'tokens/train_per_sec_per_gpu': '35.27', 'tokens/total': 90512, 'tokens/trainable': 1351, 'epoch': '0.01172'} |
|
1%| | 3/512 [00:21<1:00:02, 7.08s/it]
1%| | 4/512 [00:28<58:32, 6.91s/it]
{'loss': '0.1344', 'grad_norm': '1.834', 'learning_rate': '1.2e-05', 'ppl': '1.144', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '34.73', 'tokens/train_per_sec_per_gpu': '31.52', 'tokens/total': 120944, 'tokens/trainable': 1776, 'epoch': '0.01562'} |
|
1%| | 4/512 [00:28<58:32, 6.91s/it]
1%| | 5/512 [00:35<57:32, 6.81s/it]
{'loss': '0.139', 'grad_norm': '1.748', 'learning_rate': '1.6e-05', 'ppl': '1.149', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '35.12', 'tokens/train_per_sec_per_gpu': '37.13', 'tokens/total': 151440, 'tokens/trainable': 2259, 'epoch': '0.01953'} |
|
1%| | 5/512 [00:35<57:32, 6.81s/it]
1%| | 6/512 [00:41<57:06, 6.77s/it]
{'loss': '0.1254', 'grad_norm': '1.214', 'learning_rate': '2e-05', 'ppl': '1.134', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '36.12', 'tokens/train_per_sec_per_gpu': '33', 'tokens/total': 181984, 'tokens/trainable': 2703, 'epoch': '0.02344'} |
|
1%| | 6/512 [00:41<57:06, 6.77s/it]
1%|▏ | 7/512 [00:48<56:44, 6.74s/it]
{'loss': '0.101', 'grad_norm': '1.156', 'learning_rate': '2.4e-05', 'ppl': '1.106', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '36.12', 'tokens/train_per_sec_per_gpu': '34.03', 'tokens/total': 212336, 'tokens/trainable': 3142, 'epoch': '0.02734'} |
|
1%|▏ | 7/512 [00:48<56:44, 6.74s/it]
2%|▏ | 8/512 [00:55<56:16, 6.70s/it]
{'loss': '0.08791', 'grad_norm': '1.801', 'learning_rate': '2.8e-05', 'ppl': '1.092', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '36.12', 'tokens/train_per_sec_per_gpu': '34.66', 'tokens/total': 242592, 'tokens/trainable': 3585, 'epoch': '0.03125'} |
|
2%|▏ | 8/512 [00:55<56:16, 6.70s/it]
2%|▏ | 9/512 [01:01<55:50, 6.66s/it]
{'loss': '0.05063', 'grad_norm': '1.035', 'learning_rate': '3.2e-05', 'ppl': '1.052', 'memory/max_active (GiB)': '33.77', 'memory/max_allocated (GiB)': '33.77', 'memory/device_reserved (GiB)': '36.12', 'tokens/train_per_sec_per_gpu': '29.52', 'tokens/total': 272784, 'tokens/trainable': 4006, 'epoch': '0.03516'} |
|
2%|▏ | 9/512 [01:01<55:50, 6.66s/it]
2%|▏ | 10/512 [01:08<55:33, 6.64s/it]
{'loss': '0.02595', 'grad_norm': '1.919', 'learning_rate': '3.6e-05', 'ppl': '1.026', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '37.81', 'tokens/train_per_sec_per_gpu': '39.41', 'tokens/total': 303184, 'tokens/trainable': 4479, 'epoch': '0.03906'} |
|
2%|▏ | 10/512 [01:08<55:33, 6.64s/it]
2%|▏ | 11/512 [01:14<55:25, 6.64s/it]
{'loss': '0.04992', 'grad_norm': '2.997', 'learning_rate': '4e-05', 'ppl': '1.051', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '37.81', 'tokens/train_per_sec_per_gpu': '34.7', 'tokens/total': 333616, 'tokens/trainable': 4955, 'epoch': '0.04297'} |
|
2%|▏ | 11/512 [01:14<55:25, 6.64s/it]
2%|▏ | 12/512 [01:21<55:17, 6.63s/it]
{'loss': '0.07573', 'grad_norm': '5.029', 'learning_rate': '4.4e-05', 'ppl': '1.079', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '37.81', 'tokens/train_per_sec_per_gpu': '34.71', 'tokens/total': 364160, 'tokens/trainable': 5437, 'epoch': '0.04688'} |
|
2%|▏ | 12/512 [01:21<55:17, 6.63s/it]
3%|▎ | 13/512 [01:28<55:10, 6.64s/it]
{'loss': '0.07952', 'grad_norm': '2.657', 'learning_rate': '4.8e-05', 'ppl': '1.083', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '37.81', 'tokens/train_per_sec_per_gpu': '35.73', 'tokens/total': 394432, 'tokens/trainable': 5868, 'epoch': '0.05078'} |
|
3%|▎ | 13/512 [01:28<55:10, 6.64s/it]
3%|▎ | 14/512 [01:34<55:09, 6.65s/it]
{'loss': '0.07427', 'grad_norm': '3.637', 'learning_rate': '5.2e-05', 'ppl': '1.077', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '37.81', 'tokens/train_per_sec_per_gpu': '35.24', 'tokens/total': 424976, 'tokens/trainable': 6357, 'epoch': '0.05469'} |
|
3%|▎ | 14/512 [01:34<55:09, 6.65s/it]
3%|▎ | 15/512 [01:41<55:04, 6.65s/it]
{'loss': '0.09369', 'grad_norm': '9.169', 'learning_rate': '5.6e-05', 'ppl': '1.098', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '37.81', 'tokens/train_per_sec_per_gpu': '36.51', 'tokens/total': 455472, 'tokens/trainable': 6838, 'epoch': '0.05859'} |
|
3%|▎ | 15/512 [01:41<55:04, 6.65s/it]
3%|▎ | 16/512 [01:48<54:45, 6.62s/it]
{'loss': '0.1147', 'grad_norm': '6.009', 'learning_rate': '6e-05', 'ppl': '1.122', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '37.81', 'tokens/train_per_sec_per_gpu': '31.7', 'tokens/total': 485648, 'tokens/trainable': 7252, 'epoch': '0.0625'} |
|
3%|▎ | 16/512 [01:48<54:45, 6.62s/it]
3%|▎ | 17/512 [01:54<54:48, 6.64s/it]
{'loss': '0.02215', 'grad_norm': '1.595', 'learning_rate': '6.4e-05', 'ppl': '1.022', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '37.81', 'tokens/train_per_sec_per_gpu': '36.17', 'tokens/total': 516064, 'tokens/trainable': 7721, 'epoch': '0.06641'} |
|
3%|▎ | 17/512 [01:54<54:48, 6.64s/it]
4%|▎ | 18/512 [02:01<54:52, 6.66s/it]
{'loss': '0.04', 'grad_norm': '2.77', 'learning_rate': '6.8e-05', 'ppl': '1.041', 'memory/max_active (GiB)': '33.77', 'memory/max_allocated (GiB)': '33.77', 'memory/device_reserved (GiB)': '37.81', 'tokens/train_per_sec_per_gpu': '34.13', 'tokens/total': 546464, 'tokens/trainable': 8206, 'epoch': '0.07031'} |
|
4%|▎ | 18/512 [02:01<54:52, 6.66s/it]
4%|▎ | 19/512 [02:08<54:44, 6.66s/it]
{'loss': '0.01785', 'grad_norm': '0.9961', 'learning_rate': '7.2e-05', 'ppl': '1.018', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '37.81', 'tokens/train_per_sec_per_gpu': '35.31', 'tokens/total': 576880, 'tokens/trainable': 8697, 'epoch': '0.07422'} |
|
4%|▎ | 19/512 [02:08<54:44, 6.66s/it]
4%|▍ | 20/512 [02:14<54:38, 6.66s/it]
{'loss': '0.07808', 'grad_norm': '2.216', 'learning_rate': '7.6e-05', 'ppl': '1.081', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '37.81', 'tokens/train_per_sec_per_gpu': '40.24', 'tokens/total': 607376, 'tokens/trainable': 9211, 'epoch': '0.07812'} |
|
4%|▍ | 20/512 [02:14<54:38, 6.66s/it]
4%|▍ | 21/512 [02:21<54:34, 6.67s/it]
{'loss': '0.02783', 'grad_norm': '2.108', 'learning_rate': '8e-05', 'ppl': '1.028', 'memory/max_active (GiB)': '33.77', 'memory/max_allocated (GiB)': '33.77', 'memory/device_reserved (GiB)': '37.81', 'tokens/train_per_sec_per_gpu': '31.87', 'tokens/total': 637808, 'tokens/trainable': 9636, 'epoch': '0.08203'} |
|
4%|▍ | 21/512 [02:21<54:34, 6.67s/it]
4%|▍ | 22/512 [02:28<54:25, 6.66s/it]
{'loss': '0.03728', 'grad_norm': '89.46', 'learning_rate': '8.4e-05', 'ppl': '1.038', 'memory/max_active (GiB)': '33.75', 'memory/max_allocated (GiB)': '33.75', 'memory/device_reserved (GiB)': '37.81', 'tokens/train_per_sec_per_gpu': '36.39', 'tokens/total': 667952, 'tokens/trainable': 10108, 'epoch': '0.08594'} |
|
4%|▍ | 22/512 [02:28<54:25, 6.66s/it]
4%|▍ | 23/512 [02:34<54:21, 6.67s/it]
{'loss': '0.03399', 'grad_norm': '1.582', 'learning_rate': '8.8e-05', 'ppl': '1.035', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '37.81', 'tokens/train_per_sec_per_gpu': '33.08', 'tokens/total': 698192, 'tokens/trainable': 10557, 'epoch': '0.08984'} |
|
4%|▍ | 23/512 [02:34<54:21, 6.67s/it]
5%|▍ | 24/512 [02:41<54:11, 6.66s/it]
{'loss': '0.007771', 'grad_norm': '3.942', 'learning_rate': '9.2e-05', 'ppl': '1.008', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '37.81', 'tokens/train_per_sec_per_gpu': '31.9', 'tokens/total': 728752, 'tokens/trainable': 11015, 'epoch': '0.09375'} |
|
5%|▍ | 24/512 [02:41<54:11, 6.66s/it]
5%|▍ | 25/512 [02:47<52:43, 6.50s/it]
{'loss': '0.047', 'grad_norm': '2.1', 'learning_rate': '9.6e-05', 'ppl': '1.048', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '37.81', 'tokens/train_per_sec_per_gpu': '40.48', 'tokens/total': 756976, 'tokens/trainable': 11439, 'epoch': '0.09766'} |
|
5%|▍ | 25/512 [02:47<52:43, 6.50s/it]
5%|▌ | 26/512 [02:54<52:51, 6.53s/it]
{'loss': '0.02603', 'grad_norm': '1.807', 'learning_rate': '0.0001', 'ppl': '1.026', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '37.81', 'tokens/train_per_sec_per_gpu': '34.73', 'tokens/total': 787440, 'tokens/trainable': 11896, 'epoch': '0.1016'} |
|
5%|▌ | 26/512 [02:54<52:51, 6.53s/it]
5%|▌ | 27/512 [03:00<53:02, 6.56s/it]
{'loss': '0.01277', 'grad_norm': '0.5797', 'learning_rate': '0.0001', 'ppl': '1.013', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '37.81', 'tokens/train_per_sec_per_gpu': '32.06', 'tokens/total': 817824, 'tokens/trainable': 12338, 'epoch': '0.1055'} |
|
5%|▌ | 27/512 [03:00<53:02, 6.56s/it]
5%|▌ | 28/512 [03:07<53:10, 6.59s/it]
{'loss': '0.02406', 'grad_norm': '0.9036', 'learning_rate': '0.0001', 'ppl': '1.024', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '37.81', 'tokens/train_per_sec_per_gpu': '38.3', 'tokens/total': 848448, 'tokens/trainable': 12822, 'epoch': '0.1094'} |
|
5%|▌ | 28/512 [03:07<53:10, 6.59s/it]
6%|▌ | 29/512 [03:14<53:08, 6.60s/it]
{'loss': '0.06573', 'grad_norm': '2.259', 'learning_rate': '9.999e-05', 'ppl': '1.068', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '37.81', 'tokens/train_per_sec_per_gpu': '35.93', 'tokens/total': 878768, 'tokens/trainable': 13292, 'epoch': '0.1133'} |
|
6%|▌ | 29/512 [03:14<53:08, 6.60s/it]
6%|▌ | 30/512 [03:20<53:10, 6.62s/it]
{'loss': '0.01868', 'grad_norm': '0.786', 'learning_rate': '9.999e-05', 'ppl': '1.019', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '37.81', 'tokens/train_per_sec_per_gpu': '35.74', 'tokens/total': 909328, 'tokens/trainable': 13781, 'epoch': '0.1172'} |
|
6%|▌ | 30/512 [03:20<53:10, 6.62s/it]
6%|▌ | 31/512 [03:27<53:10, 6.63s/it]
{'loss': '0.02133', 'grad_norm': '1.028', 'learning_rate': '9.998e-05', 'ppl': '1.022', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '37.81', 'tokens/train_per_sec_per_gpu': '34.35', 'tokens/total': 939632, 'tokens/trainable': 14247, 'epoch': '0.1211'} |
|
6%|▌ | 31/512 [03:27<53:10, 6.63s/it]
6%|▋ | 32/512 [03:34<52:50, 6.61s/it]
{'loss': '0.01562', 'grad_norm': '0.7983', 'learning_rate': '9.997e-05', 'ppl': '1.016', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '37.81', 'tokens/train_per_sec_per_gpu': '34.87', 'tokens/total': 969584, 'tokens/trainable': 14677, 'epoch': '0.125'} |
|
6%|▋ | 32/512 [03:34<52:50, 6.61s/it][2026-08-18 14:35:17,170] [INFO] [axolotl.core.trainers.base._save:828] [PID:12035] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-32 |
|
6%|▋ | 33/512 [03:42<57:31, 7.20s/it]
{'loss': '0.04403', 'grad_norm': '0.9356', 'learning_rate': '9.995e-05', 'ppl': '1.045', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '37.81', 'tokens/train_per_sec_per_gpu': '33.71', 'tokens/total': 1000112, 'tokens/trainable': 15109, 'epoch': '0.1289'} |
|
6%|▋ | 33/512 [03:42<57:31, 7.20s/it]
7%|▋ | 34/512 [03:49<55:51, 7.01s/it]
{'loss': '0.02189', 'grad_norm': '0.5472', 'learning_rate': '9.994e-05', 'ppl': '1.022', 'memory/max_active (GiB)': '33.77', 'memory/max_allocated (GiB)': '33.77', 'memory/device_reserved (GiB)': '35.48', 'tokens/train_per_sec_per_gpu': '34.32', 'tokens/total': 1030160, 'tokens/trainable': 15570, 'epoch': '0.1328'} |
|
7%|▋ | 34/512 [03:49<55:51, 7.01s/it]
7%|▋ | 35/512 [03:55<54:55, 6.91s/it]
{'loss': '0.01708', 'grad_norm': '0.5502', 'learning_rate': '9.992e-05', 'ppl': '1.017', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.48', 'tokens/train_per_sec_per_gpu': '34.18', 'tokens/total': 1060736, 'tokens/trainable': 16059, 'epoch': '0.1367'} |
|
7%|▋ | 35/512 [03:55<54:55, 6.91s/it]
7%|▋ | 36/512 [04:02<53:57, 6.80s/it]
{'loss': '0.01175', 'grad_norm': '1.768', 'learning_rate': '9.991e-05', 'ppl': '1.012', 'memory/max_active (GiB)': '33.72', 'memory/max_allocated (GiB)': '33.72', 'memory/device_reserved (GiB)': '35.48', 'tokens/train_per_sec_per_gpu': '31.46', 'tokens/total': 1090784, 'tokens/trainable': 16497, 'epoch': '0.1406'} |
|
7%|▋ | 36/512 [04:02<53:57, 6.80s/it]
7%|▋ | 37/512 [04:08<53:19, 6.74s/it]
{'loss': '0.05863', 'grad_norm': '1.32', 'learning_rate': '9.989e-05', 'ppl': '1.06', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '35.86', 'tokens/train_per_sec_per_gpu': '34.79', 'tokens/total': 1121264, 'tokens/trainable': 16933, 'epoch': '0.1445'} |
|
7%|▋ | 37/512 [04:08<53:19, 6.74s/it]
7%|▋ | 38/512 [04:15<52:54, 6.70s/it]
{'loss': '0.07118', 'grad_norm': '2.745', 'learning_rate': '9.987e-05', 'ppl': '1.074', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '35.86', 'tokens/train_per_sec_per_gpu': '29.21', 'tokens/total': 1151680, 'tokens/trainable': 17354, 'epoch': '0.1484'} |
|
7%|▋ | 38/512 [04:15<52:54, 6.70s/it]
8%|▊ | 39/512 [04:22<52:44, 6.69s/it]
{'loss': '0.03757', 'grad_norm': '0.8014', 'learning_rate': '9.984e-05', 'ppl': '1.038', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.86', 'tokens/train_per_sec_per_gpu': '36.59', 'tokens/total': 1182048, 'tokens/trainable': 17827, 'epoch': '0.1523'} |
|
8%|▊ | 39/512 [04:22<52:44, 6.69s/it]
8%|▊ | 40/512 [04:28<52:29, 6.67s/it]
{'loss': '0.0387', 'grad_norm': '1.031', 'learning_rate': '9.982e-05', 'ppl': '1.039', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '35.86', 'tokens/train_per_sec_per_gpu': '36.04', 'tokens/total': 1212384, 'tokens/trainable': 18311, 'epoch': '0.1562'} |
|
8%|▊ | 40/512 [04:28<52:29, 6.67s/it]
8%|▊ | 41/512 [04:35<52:13, 6.65s/it]
{'loss': '0.05204', 'grad_norm': '2.123', 'learning_rate': '9.979e-05', 'ppl': '1.053', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '32.4', 'tokens/total': 1242912, 'tokens/trainable': 18768, 'epoch': '0.1602'} |
|
8%|▊ | 41/512 [04:35<52:13, 6.65s/it]
8%|▊ | 42/512 [04:42<52:01, 6.64s/it]
{'loss': '0.04878', 'grad_norm': '0.6702', 'learning_rate': '9.976e-05', 'ppl': '1.05', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '32.22', 'tokens/total': 1273120, 'tokens/trainable': 19193, 'epoch': '0.1641'} |
|
8%|▊ | 42/512 [04:42<52:01, 6.64s/it]
8%|▊ | 43/512 [04:48<52:03, 6.66s/it]
{'loss': '0.009711', 'grad_norm': '0.2514', 'learning_rate': '9.973e-05', 'ppl': '1.01', 'memory/max_active (GiB)': '34.04', 'memory/max_allocated (GiB)': '34.04', 'memory/device_reserved (GiB)': '36.03', 'tokens/train_per_sec_per_gpu': '31.64', 'tokens/total': 1303920, 'tokens/trainable': 19643, 'epoch': '0.168'} |
|
8%|▊ | 43/512 [04:48<52:03, 6.66s/it]
9%|▊ | 44/512 [04:55<51:49, 6.64s/it]
{'loss': '0.03127', 'grad_norm': '0.4949', 'learning_rate': '9.97e-05', 'ppl': '1.032', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '36.04', 'tokens/train_per_sec_per_gpu': '33.75', 'tokens/total': 1334256, 'tokens/trainable': 20093, 'epoch': '0.1719'} |
|
9%|▊ | 44/512 [04:55<51:49, 6.64s/it]
9%|▉ | 45/512 [05:02<51:40, 6.64s/it]
{'loss': '0.03356', 'grad_norm': '0.4885', 'learning_rate': '9.966e-05', 'ppl': '1.034', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '36.04', 'tokens/train_per_sec_per_gpu': '31.84', 'tokens/total': 1364608, 'tokens/trainable': 20524, 'epoch': '0.1758'} |
|
9%|▉ | 45/512 [05:02<51:40, 6.64s/it]
9%|▉ | 46/512 [05:08<51:30, 6.63s/it]
{'loss': '0.02636', 'grad_norm': '2.134', 'learning_rate': '9.963e-05', 'ppl': '1.027', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '36.04', 'tokens/train_per_sec_per_gpu': '34.64', 'tokens/total': 1394688, 'tokens/trainable': 20967, 'epoch': '0.1797'} |
|
9%|▉ | 46/512 [05:08<51:30, 6.63s/it]
9%|▉ | 47/512 [05:15<51:31, 6.65s/it]
{'loss': '0.01183', 'grad_norm': '0.3662', 'learning_rate': '9.959e-05', 'ppl': '1.012', 'memory/max_active (GiB)': '33.98', 'memory/max_allocated (GiB)': '33.98', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '35.28', 'tokens/total': 1425344, 'tokens/trainable': 21431, 'epoch': '0.1836'} |
|
9%|▉ | 47/512 [05:15<51:31, 6.65s/it]
9%|▉ | 48/512 [05:21<51:18, 6.63s/it]
{'loss': '0.01725', 'grad_norm': '0.951', 'learning_rate': '9.955e-05', 'ppl': '1.017', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '35.46', 'tokens/total': 1455712, 'tokens/trainable': 21854, 'epoch': '0.1875'} |
|
9%|▉ | 48/512 [05:21<51:18, 6.63s/it]
10%|▉ | 49/512 [05:28<49:51, 6.46s/it]
{'loss': '0.02522', 'grad_norm': '4.225', 'learning_rate': '9.951e-05', 'ppl': '1.026', 'memory/max_active (GiB)': '33.29', 'memory/max_allocated (GiB)': '33.29', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '32.38', 'tokens/total': 1483760, 'tokens/trainable': 22269, 'epoch': '0.1914'} |
|
10%|▉ | 49/512 [05:28<49:51, 6.46s/it]
10%|▉ | 50/512 [05:34<50:05, 6.51s/it]
{'loss': '0.01367', 'grad_norm': '0.4782', 'learning_rate': '9.946e-05', 'ppl': '1.014', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '36.75', 'tokens/total': 1514128, 'tokens/trainable': 22716, 'epoch': '0.1953'} |
|
10%|▉ | 50/512 [05:34<50:05, 6.51s/it]
10%|▉ | 51/512 [05:41<50:08, 6.53s/it]
{'loss': '0.003458', 'grad_norm': '0.2308', 'learning_rate': '9.942e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '32.41', 'tokens/total': 1544368, 'tokens/trainable': 23164, 'epoch': '0.1992'} |
|
10%|▉ | 51/512 [05:41<50:08, 6.53s/it]
10%|█ | 52/512 [05:47<50:19, 6.56s/it]
{'loss': '0.01783', 'grad_norm': '0.5431', 'learning_rate': '9.937e-05', 'ppl': '1.018', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '37.31', 'tokens/total': 1574672, 'tokens/trainable': 23659, 'epoch': '0.2031'} |
|
10%|█ | 52/512 [05:47<50:19, 6.56s/it]
10%|█ | 53/512 [05:54<50:23, 6.59s/it]
{'loss': '0.03821', 'grad_norm': '1.049', 'learning_rate': '9.932e-05', 'ppl': '1.039', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '34.65', 'tokens/total': 1604912, 'tokens/trainable': 24125, 'epoch': '0.207'} |
|
10%|█ | 53/512 [05:54<50:23, 6.59s/it]
11%|█ | 54/512 [06:01<50:12, 6.58s/it]
{'loss': '0.02056', 'grad_norm': '0.986', 'learning_rate': '9.927e-05', 'ppl': '1.021', 'memory/max_active (GiB)': '33.66', 'memory/max_allocated (GiB)': '33.66', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '38.29', 'tokens/total': 1634784, 'tokens/trainable': 24599, 'epoch': '0.2109'} |
|
11%|█ | 54/512 [06:01<50:12, 6.58s/it]
11%|█ | 55/512 [06:07<50:18, 6.61s/it]
{'loss': '0.02877', 'grad_norm': '1.086', 'learning_rate': '9.921e-05', 'ppl': '1.029', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '34.33', 'tokens/total': 1665280, 'tokens/trainable': 25096, 'epoch': '0.2148'} |
|
11%|█ | 55/512 [06:07<50:18, 6.61s/it]
11%|█ | 56/512 [06:14<50:16, 6.61s/it]
{'loss': '0.01534', 'grad_norm': '0.5943', 'learning_rate': '9.916e-05', 'ppl': '1.015', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '36.94', 'tokens/total': 1695616, 'tokens/trainable': 25570, 'epoch': '0.2188'} |
|
11%|█ | 56/512 [06:14<50:16, 6.61s/it]
11%|█ | 57/512 [06:20<50:10, 6.62s/it]
{'loss': '0.02529', 'grad_norm': '1.077', 'learning_rate': '9.91e-05', 'ppl': '1.026', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '33.53', 'tokens/total': 1725824, 'tokens/trainable': 25994, 'epoch': '0.2227'} |
|
11%|█ | 57/512 [06:20<50:10, 6.62s/it]
11%|█▏ | 58/512 [06:27<50:02, 6.61s/it]
{'loss': '0.01261', 'grad_norm': '0.7194', 'learning_rate': '9.904e-05', 'ppl': '1.013', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '34.24', 'tokens/total': 1756336, 'tokens/trainable': 26438, 'epoch': '0.2266'} |
|
11%|█▏ | 58/512 [06:27<50:02, 6.61s/it]
12%|█▏ | 59/512 [06:34<50:03, 6.63s/it]
{'loss': '0.02943', 'grad_norm': '0.7495', 'learning_rate': '9.898e-05', 'ppl': '1.03', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '34.33', 'tokens/total': 1786960, 'tokens/trainable': 26912, 'epoch': '0.2305'} |
|
12%|█▏ | 59/512 [06:34<50:03, 6.63s/it]
12%|█▏ | 60/512 [06:40<50:03, 6.65s/it]
{'loss': '0.02261', 'grad_norm': '0.5475', 'learning_rate': '9.892e-05', 'ppl': '1.023', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '33.84', 'tokens/total': 1817200, 'tokens/trainable': 27352, 'epoch': '0.2344'} |
|
12%|█▏ | 60/512 [06:40<50:03, 6.65s/it]
12%|█▏ | 61/512 [06:47<50:04, 6.66s/it]
{'loss': '0.0044', 'grad_norm': '0.1943', 'learning_rate': '9.886e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '39.28', 'tokens/total': 1847808, 'tokens/trainable': 27859, 'epoch': '0.2383'} |
|
12%|█▏ | 61/512 [06:47<50:04, 6.66s/it]
12%|█▏ | 62/512 [06:54<50:08, 6.69s/it]
{'loss': '0.02336', 'grad_norm': '0.7837', 'learning_rate': '9.879e-05', 'ppl': '1.024', 'memory/max_active (GiB)': '34.02', 'memory/max_allocated (GiB)': '34.02', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '32.65', 'tokens/total': 1878416, 'tokens/trainable': 28310, 'epoch': '0.2422'} |
|
12%|█▏ | 62/512 [06:54<50:08, 6.69s/it]
12%|█▏ | 63/512 [07:00<49:53, 6.67s/it]
{'loss': '0.01773', 'grad_norm': '0.6597', 'learning_rate': '9.872e-05', 'ppl': '1.018', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '32.77', 'tokens/total': 1908896, 'tokens/trainable': 28772, 'epoch': '0.2461'} |
|
12%|█▏ | 63/512 [07:00<49:53, 6.67s/it]
12%|█▎ | 64/512 [07:07<49:44, 6.66s/it]
{'loss': '0.03213', 'grad_norm': '0.8329', 'learning_rate': '9.865e-05', 'ppl': '1.033', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '33.54', 'tokens/total': 1939200, 'tokens/trainable': 29220, 'epoch': '0.25'} |
|
12%|█▎ | 64/512 [07:07<49:44, 6.66s/it][2026-08-18 14:38:50,796] [INFO] [axolotl.core.trainers.base._save:828] [PID:12035] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-64 |
|
13%|█▎ | 65/512 [07:15<53:24, 7.17s/it]
{'loss': '0.01292', 'grad_norm': '0.3883', 'learning_rate': '9.858e-05', 'ppl': '1.013', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '31.88', 'tokens/total': 1969696, 'tokens/trainable': 29685, 'epoch': '0.2539'} |
|
13%|█▎ | 65/512 [07:15<53:24, 7.17s/it]
13%|█▎ | 66/512 [07:22<52:00, 7.00s/it]
{'loss': '0.0107', 'grad_norm': '0.493', 'learning_rate': '9.851e-05', 'ppl': '1.011', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '35.06', 'tokens/train_per_sec_per_gpu': '33.81', 'tokens/total': 2000064, 'tokens/trainable': 30112, 'epoch': '0.2578'} |
|
13%|█▎ | 66/512 [07:22<52:00, 7.00s/it]
13%|█▎ | 67/512 [07:28<50:03, 6.75s/it]
{'loss': '0.03524', 'grad_norm': '0.4086', 'learning_rate': '9.844e-05', 'ppl': '1.036', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '35.65', 'tokens/train_per_sec_per_gpu': '37.93', 'tokens/total': 2028288, 'tokens/trainable': 30591, 'epoch': '0.2617'} |
|
13%|█▎ | 67/512 [07:28<50:03, 6.75s/it]
13%|█▎ | 68/512 [07:35<49:43, 6.72s/it]
{'loss': '0.003894', 'grad_norm': '0.2266', 'learning_rate': '9.836e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.65', 'tokens/train_per_sec_per_gpu': '35.49', 'tokens/total': 2058736, 'tokens/trainable': 31070, 'epoch': '0.2656'} |
|
13%|█▎ | 68/512 [07:35<49:43, 6.72s/it]
13%|█▎ | 69/512 [07:42<49:29, 6.70s/it]
{'loss': '0.02871', 'grad_norm': '1.044', 'learning_rate': '9.828e-05', 'ppl': '1.029', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.65', 'tokens/train_per_sec_per_gpu': '34.23', 'tokens/total': 2089040, 'tokens/trainable': 31550, 'epoch': '0.2695'} |
|
13%|█▎ | 69/512 [07:42<49:29, 6.70s/it]
14%|█▎ | 70/512 [07:48<49:08, 6.67s/it]
{'loss': '0.008445', 'grad_norm': '0.5291', 'learning_rate': '9.82e-05', 'ppl': '1.008', 'memory/max_active (GiB)': '33.77', 'memory/max_allocated (GiB)': '33.77', 'memory/device_reserved (GiB)': '35.65', 'tokens/train_per_sec_per_gpu': '32.6', 'tokens/total': 2119200, 'tokens/trainable': 32001, 'epoch': '0.2734'} |
|
14%|█▎ | 70/512 [07:48<49:08, 6.67s/it]
14%|█▍ | 71/512 [07:54<47:45, 6.50s/it]
{'loss': '0.008676', 'grad_norm': '0.3681', 'learning_rate': '9.812e-05', 'ppl': '1.009', 'memory/max_active (GiB)': '33.35', 'memory/max_allocated (GiB)': '33.35', 'memory/device_reserved (GiB)': '35.65', 'tokens/train_per_sec_per_gpu': '32.35', 'tokens/total': 2147440, 'tokens/trainable': 32417, 'epoch': '0.2773'} |
|
14%|█▍ | 71/512 [07:54<47:45, 6.50s/it]
14%|█▍ | 72/512 [08:01<47:54, 6.53s/it]
{'loss': '0.006214', 'grad_norm': '1.056', 'learning_rate': '9.803e-05', 'ppl': '1.006', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '35.65', 'tokens/train_per_sec_per_gpu': '34.94', 'tokens/total': 2177792, 'tokens/trainable': 32876, 'epoch': '0.2812'} |
|
14%|█▍ | 72/512 [08:01<47:54, 6.53s/it]
14%|█▍ | 73/512 [08:07<47:55, 6.55s/it]
{'loss': '0.02896', 'grad_norm': '1.171', 'learning_rate': '9.795e-05', 'ppl': '1.029', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '35.65', 'tokens/train_per_sec_per_gpu': '36.11', 'tokens/total': 2208384, 'tokens/trainable': 33347, 'epoch': '0.2852'} |
|
14%|█▍ | 73/512 [08:07<47:55, 6.55s/it]
14%|█▍ | 74/512 [08:14<48:11, 6.60s/it]
{'loss': '0.0214', 'grad_norm': '1.029', 'learning_rate': '9.786e-05', 'ppl': '1.022', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '35.65', 'tokens/train_per_sec_per_gpu': '32.74', 'tokens/total': 2238912, 'tokens/trainable': 33814, 'epoch': '0.2891'} |
|
14%|█▍ | 74/512 [08:14<48:11, 6.60s/it]
15%|█▍ | 75/512 [08:21<48:02, 6.60s/it]
{'loss': '0.01587', 'grad_norm': '0.9864', 'learning_rate': '9.777e-05', 'ppl': '1.016', 'memory/max_active (GiB)': '33.74', 'memory/max_allocated (GiB)': '33.74', 'memory/device_reserved (GiB)': '35.65', 'tokens/train_per_sec_per_gpu': '34.79', 'tokens/total': 2269008, 'tokens/trainable': 34228, 'epoch': '0.293'} |
|
15%|█▍ | 75/512 [08:21<48:02, 6.60s/it]
15%|█▍ | 76/512 [08:27<48:05, 6.62s/it]
{'loss': '0.01204', 'grad_norm': '0.9208', 'learning_rate': '9.768e-05', 'ppl': '1.012', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '35.65', 'tokens/train_per_sec_per_gpu': '36.44', 'tokens/total': 2299264, 'tokens/trainable': 34717, 'epoch': '0.2969'} |
|
15%|█▍ | 76/512 [08:27<48:05, 6.62s/it]
15%|█▌ | 77/512 [08:34<47:56, 6.61s/it]
{'loss': '0.03252', 'grad_norm': '0.7358', 'learning_rate': '9.759e-05', 'ppl': '1.033', 'memory/max_active (GiB)': '33.76', 'memory/max_allocated (GiB)': '33.76', 'memory/device_reserved (GiB)': '35.65', 'tokens/train_per_sec_per_gpu': '31.99', 'tokens/total': 2329360, 'tokens/trainable': 35144, 'epoch': '0.3008'} |
|
15%|█▌ | 77/512 [08:34<47:56, 6.61s/it]
15%|█▌ | 78/512 [08:41<48:05, 6.65s/it]
{'loss': '0.006883', 'grad_norm': '0.3902', 'learning_rate': '9.749e-05', 'ppl': '1.007', 'memory/max_active (GiB)': '33.99', 'memory/max_allocated (GiB)': '33.99', 'memory/device_reserved (GiB)': '35.65', 'tokens/train_per_sec_per_gpu': '35.06', 'tokens/total': 2359872, 'tokens/trainable': 35608, 'epoch': '0.3047'} |
|
15%|█▌ | 78/512 [08:41<48:05, 6.65s/it]
15%|█▌ | 79/512 [08:47<47:56, 6.64s/it]
{'loss': '0.01565', 'grad_norm': '0.658', 'learning_rate': '9.74e-05', 'ppl': '1.016', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.65', 'tokens/train_per_sec_per_gpu': '29.74', 'tokens/total': 2390224, 'tokens/trainable': 36035, 'epoch': '0.3086'} |
|
15%|█▌ | 79/512 [08:47<47:56, 6.64s/it]
16%|█▌ | 80/512 [08:54<47:53, 6.65s/it]
{'loss': '0.01866', 'grad_norm': '0.5285', 'learning_rate': '9.73e-05', 'ppl': '1.019', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.65', 'tokens/train_per_sec_per_gpu': '38.53', 'tokens/total': 2420720, 'tokens/trainable': 36520, 'epoch': '0.3125'} |
|
16%|█▌ | 80/512 [08:54<47:53, 6.65s/it]
16%|█▌ | 81/512 [09:01<47:42, 6.64s/it]
{'loss': '0.01531', 'grad_norm': '1.149', 'learning_rate': '9.72e-05', 'ppl': '1.015', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '33.57', 'tokens/total': 2451040, 'tokens/trainable': 36977, 'epoch': '0.3164'} |
|
16%|█▌ | 81/512 [09:01<47:42, 6.64s/it]
16%|█▌ | 82/512 [09:07<46:31, 6.49s/it]
{'loss': '0.01632', 'grad_norm': '0.5861', 'learning_rate': '9.71e-05', 'ppl': '1.016', 'memory/max_active (GiB)': '33.44', 'memory/max_allocated (GiB)': '33.44', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '33.87', 'tokens/total': 2479504, 'tokens/trainable': 37437, 'epoch': '0.3203'} |
|
16%|█▌ | 82/512 [09:07<46:31, 6.49s/it]
16%|█▌ | 83/512 [09:13<46:43, 6.54s/it]
{'loss': '0.01133', 'grad_norm': '0.5118', 'learning_rate': '9.699e-05', 'ppl': '1.011', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '33.45', 'tokens/total': 2509776, 'tokens/trainable': 37889, 'epoch': '0.3242'} |
|
16%|█▌ | 83/512 [09:13<46:43, 6.54s/it]
16%|█▋ | 84/512 [09:20<46:55, 6.58s/it]
{'loss': '0.0212', 'grad_norm': '0.5954', 'learning_rate': '9.689e-05', 'ppl': '1.021', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '32.97', 'tokens/total': 2540272, 'tokens/trainable': 38352, 'epoch': '0.3281'} |
|
16%|█▋ | 84/512 [09:20<46:55, 6.58s/it]
17%|█▋ | 85/512 [09:27<46:59, 6.60s/it]
{'loss': '0.004736', 'grad_norm': '0.2197', 'learning_rate': '9.678e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.69', 'memory/max_allocated (GiB)': '33.69', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '33.79', 'tokens/total': 2570512, 'tokens/trainable': 38807, 'epoch': '0.332'} |
|
17%|█▋ | 85/512 [09:27<46:59, 6.60s/it]
17%|█▋ | 86/512 [09:33<46:57, 6.61s/it]
{'loss': '0.03325', 'grad_norm': '1.302', 'learning_rate': '9.667e-05', 'ppl': '1.034', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '30.74', 'tokens/total': 2600992, 'tokens/trainable': 39235, 'epoch': '0.3359'} |
|
17%|█▋ | 86/512 [09:33<46:57, 6.61s/it]
17%|█▋ | 87/512 [09:40<46:55, 6.63s/it]
{'loss': '0.03753', 'grad_norm': '0.9726', 'learning_rate': '9.656e-05', 'ppl': '1.038', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '36.84', 'tokens/total': 2631360, 'tokens/trainable': 39743, 'epoch': '0.3398'} |
|
17%|█▋ | 87/512 [09:40<46:55, 6.63s/it]
17%|█▋ | 88/512 [09:47<46:40, 6.61s/it]
{'loss': '0.02125', 'grad_norm': '0.5336', 'learning_rate': '9.645e-05', 'ppl': '1.021', 'memory/max_active (GiB)': '33.73', 'memory/max_allocated (GiB)': '33.73', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '33.09', 'tokens/total': 2661520, 'tokens/trainable': 40183, 'epoch': '0.3438'} |
|
17%|█▋ | 88/512 [09:47<46:40, 6.61s/it]
17%|█▋ | 89/512 [09:53<46:37, 6.61s/it]
{'loss': '0.02684', 'grad_norm': '0.4704', 'learning_rate': '9.633e-05', 'ppl': '1.027', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '33.04', 'tokens/total': 2691984, 'tokens/trainable': 40627, 'epoch': '0.3477'} |
|
17%|█▋ | 89/512 [09:53<46:37, 6.61s/it]
18%|█▊ | 90/512 [10:00<46:32, 6.62s/it]
{'loss': '0.0152', 'grad_norm': '0.3449', 'learning_rate': '9.622e-05', 'ppl': '1.015', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '31.09', 'tokens/total': 2722272, 'tokens/trainable': 41045, 'epoch': '0.3516'} |
|
18%|█▊ | 90/512 [10:00<46:32, 6.62s/it]
18%|█▊ | 91/512 [10:07<46:30, 6.63s/it]
{'loss': '0.01411', 'grad_norm': '0.6914', 'learning_rate': '9.61e-05', 'ppl': '1.014', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '36.84', 'tokens/total': 2752528, 'tokens/trainable': 41510, 'epoch': '0.3555'} |
|
18%|█▊ | 91/512 [10:07<46:30, 6.63s/it]
18%|█▊ | 92/512 [10:13<46:19, 6.62s/it]
{'loss': '0.01653', 'grad_norm': '0.4882', 'learning_rate': '9.598e-05', 'ppl': '1.017', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '36.09', 'tokens/total': 2782960, 'tokens/trainable': 41967, 'epoch': '0.3594'} |
|
18%|█▊ | 92/512 [10:13<46:19, 6.62s/it]
18%|█▊ | 93/512 [10:20<46:16, 6.63s/it]
{'loss': '0.02178', 'grad_norm': '1.121', 'learning_rate': '9.586e-05', 'ppl': '1.022', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '35.23', 'tokens/total': 2813408, 'tokens/trainable': 42437, 'epoch': '0.3633'} |
|
18%|█▊ | 93/512 [10:20<46:16, 6.63s/it]
18%|█▊ | 94/512 [10:26<46:12, 6.63s/it]
{'loss': '0.006203', 'grad_norm': '0.201', 'learning_rate': '9.574e-05', 'ppl': '1.006', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '37', 'tokens/total': 2843888, 'tokens/trainable': 42923, 'epoch': '0.3672'} |
|
18%|█▊ | 94/512 [10:26<46:12, 6.63s/it]
19%|█▊ | 95/512 [10:33<46:10, 6.64s/it]
{'loss': '0.006227', 'grad_norm': '0.2573', 'learning_rate': '9.562e-05', 'ppl': '1.006', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '39.16', 'tokens/total': 2874240, 'tokens/trainable': 43421, 'epoch': '0.3711'} |
|
19%|█▊ | 95/512 [10:33<46:10, 6.64s/it]
19%|█▉ | 96/512 [10:40<46:04, 6.65s/it]
{'loss': '0.01506', 'grad_norm': '0.5242', 'learning_rate': '9.549e-05', 'ppl': '1.015', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '33.4', 'tokens/total': 2904576, 'tokens/trainable': 43864, 'epoch': '0.375'} |
|
19%|█▉ | 96/512 [10:40<46:04, 6.65s/it][2026-08-18 14:42:23,439] [INFO] [axolotl.core.trainers.base._save:828] [PID:12035] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-96 |
|
19%|█▉ | 97/512 [10:48<49:13, 7.12s/it]
{'loss': '0.00706', 'grad_norm': '0.4452', 'learning_rate': '9.536e-05', 'ppl': '1.007', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '31.91', 'tokens/total': 2935040, 'tokens/trainable': 44295, 'epoch': '0.3789'} |
|
19%|█▉ | 97/512 [10:48<49:13, 7.12s/it]
19%|█▉ | 98/512 [10:55<48:08, 6.98s/it]
{'loss': '0.06137', 'grad_norm': '1.164', 'learning_rate': '9.523e-05', 'ppl': '1.063', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '35.19', 'tokens/train_per_sec_per_gpu': '39.11', 'tokens/total': 2965360, 'tokens/trainable': 44786, 'epoch': '0.3828'} |
|
19%|█▉ | 98/512 [10:55<48:08, 6.98s/it]
19%|█▉ | 99/512 [11:01<47:36, 6.92s/it]
{'loss': '0.005952', 'grad_norm': '0.5285', 'learning_rate': '9.51e-05', 'ppl': '1.006', 'memory/max_active (GiB)': '34.03', 'memory/max_allocated (GiB)': '34.03', 'memory/device_reserved (GiB)': '35.76', 'tokens/train_per_sec_per_gpu': '36.34', 'tokens/total': 2995952, 'tokens/trainable': 45249, 'epoch': '0.3867'} |
|
19%|█▉ | 99/512 [11:01<47:36, 6.92s/it]
20%|█▉ | 100/512 [11:08<46:53, 6.83s/it]
{'loss': '0.01775', 'grad_norm': '0.4024', 'learning_rate': '9.497e-05', 'ppl': '1.018', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '33.35', 'tokens/total': 3026320, 'tokens/trainable': 45693, 'epoch': '0.3906'} |
|
20%|█▉ | 100/512 [11:08<46:53, 6.83s/it]
20%|█▉ | 101/512 [11:15<46:22, 6.77s/it]
{'loss': '0.01136', 'grad_norm': '0.3559', 'learning_rate': '9.484e-05', 'ppl': '1.011', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '33.5', 'tokens/total': 3056560, 'tokens/trainable': 46136, 'epoch': '0.3945'} |
|
20%|█▉ | 101/512 [11:15<46:22, 6.77s/it]
20%|█▉ | 102/512 [11:21<46:02, 6.74s/it]
{'loss': '0.0115', 'grad_norm': '0.4545', 'learning_rate': '9.47e-05', 'ppl': '1.012', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '32.88', 'tokens/total': 3087136, 'tokens/trainable': 46592, 'epoch': '0.3984'} |
|
20%|█▉ | 102/512 [11:21<46:02, 6.74s/it]
20%|██ | 103/512 [11:28<45:46, 6.71s/it]
{'loss': '0.01274', 'grad_norm': '0.4275', 'learning_rate': '9.456e-05', 'ppl': '1.013', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '32.13', 'tokens/total': 3117712, 'tokens/trainable': 47059, 'epoch': '0.4023'} |
|
20%|██ | 103/512 [11:28<45:46, 6.71s/it]
20%|██ | 104/512 [11:35<45:33, 6.70s/it]
{'loss': '0.03431', 'grad_norm': '0.9903', 'learning_rate': '9.442e-05', 'ppl': '1.035', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '37.98', 'tokens/total': 3147984, 'tokens/trainable': 47548, 'epoch': '0.4062'} |
|
20%|██ | 104/512 [11:35<45:33, 6.70s/it]
21%|██ | 105/512 [11:41<45:24, 6.69s/it]
{'loss': '0.01038', 'grad_norm': '0.2757', 'learning_rate': '9.428e-05', 'ppl': '1.01', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '36.54', 'tokens/total': 3178352, 'tokens/trainable': 48041, 'epoch': '0.4102'} |
|
21%|██ | 105/512 [11:41<45:24, 6.69s/it]
21%|██ | 106/512 [11:47<44:08, 6.52s/it]
{'loss': '0.01873', 'grad_norm': '0.3845', 'learning_rate': '9.414e-05', 'ppl': '1.019', 'memory/max_active (GiB)': '33.39', 'memory/max_allocated (GiB)': '33.39', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '34.46', 'tokens/total': 3206608, 'tokens/trainable': 48492, 'epoch': '0.4141'} |
|
21%|██ | 106/512 [11:47<44:08, 6.52s/it]
21%|██ | 107/512 [11:54<44:10, 6.54s/it]
{'loss': '0.03059', 'grad_norm': '0.6006', 'learning_rate': '9.4e-05', 'ppl': '1.031', 'memory/max_active (GiB)': '33.74', 'memory/max_allocated (GiB)': '33.74', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '31.11', 'tokens/total': 3236704, 'tokens/trainable': 48926, 'epoch': '0.418'} |
|
21%|██ | 107/512 [11:54<44:10, 6.54s/it]
21%|██ | 108/512 [12:01<44:12, 6.57s/it]
{'loss': '0.01344', 'grad_norm': '0.2173', 'learning_rate': '9.385e-05', 'ppl': '1.014', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '38.4', 'tokens/total': 3267072, 'tokens/trainable': 49397, 'epoch': '0.4219'} |
|
21%|██ | 108/512 [12:01<44:12, 6.57s/it]
21%|██▏ | 109/512 [12:07<44:13, 6.58s/it]
{'loss': '0.02159', 'grad_norm': '0.2254', 'learning_rate': '9.37e-05', 'ppl': '1.022', 'memory/max_active (GiB)': '33.79', 'memory/max_allocated (GiB)': '33.79', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '37.26', 'tokens/total': 3297200, 'tokens/trainable': 49864, 'epoch': '0.4258'} |
|
21%|██▏ | 109/512 [12:07<44:13, 6.58s/it]
21%|██▏ | 110/512 [12:14<44:18, 6.61s/it]
{'loss': '0.02149', 'grad_norm': '0.3758', 'learning_rate': '9.355e-05', 'ppl': '1.022', 'memory/max_active (GiB)': '33.99', 'memory/max_allocated (GiB)': '33.99', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '33.24', 'tokens/total': 3327616, 'tokens/trainable': 50303, 'epoch': '0.4297'} |
|
21%|██▏ | 110/512 [12:14<44:18, 6.61s/it]
22%|██▏ | 111/512 [12:21<44:13, 6.62s/it]
{'loss': '0.02922', 'grad_norm': '0.2576', 'learning_rate': '9.34e-05', 'ppl': '1.03', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '33.8', 'tokens/total': 3357824, 'tokens/trainable': 50746, 'epoch': '0.4336'} |
|
22%|██▏ | 111/512 [12:21<44:13, 6.62s/it]
22%|██▏ | 112/512 [12:27<44:06, 6.62s/it]
{'loss': '0.02274', 'grad_norm': '0.3464', 'learning_rate': '9.325e-05', 'ppl': '1.023', 'memory/max_active (GiB)': '33.74', 'memory/max_allocated (GiB)': '33.74', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '31.91', 'tokens/total': 3387856, 'tokens/trainable': 51192, 'epoch': '0.4375'} |
|
22%|██▏ | 112/512 [12:27<44:06, 6.62s/it]
22%|██▏ | 113/512 [12:34<44:05, 6.63s/it]
{'loss': '0.01231', 'grad_norm': '0.3253', 'learning_rate': '9.31e-05', 'ppl': '1.012', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '33.04', 'tokens/total': 3418304, 'tokens/trainable': 51648, 'epoch': '0.4414'} |
|
22%|██▏ | 113/512 [12:34<44:05, 6.63s/it]
22%|██▏ | 114/512 [12:40<43:54, 6.62s/it]
{'loss': '0.01244', 'grad_norm': '0.3918', 'learning_rate': '9.294e-05', 'ppl': '1.013', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '30.79', 'tokens/total': 3448560, 'tokens/trainable': 52061, 'epoch': '0.4453'} |
|
22%|██▏ | 114/512 [12:40<43:54, 6.62s/it]
22%|██▏ | 115/512 [12:47<43:47, 6.62s/it]
{'loss': '0.006095', 'grad_norm': '0.1873', 'learning_rate': '9.278e-05', 'ppl': '1.006', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '36.72', 'tokens/total': 3478928, 'tokens/trainable': 52550, 'epoch': '0.4492'} |
|
22%|██▏ | 115/512 [12:47<43:47, 6.62s/it]
23%|██▎ | 116/512 [12:54<43:44, 6.63s/it]
{'loss': '0.002756', 'grad_norm': '0.1563', 'learning_rate': '9.263e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '38.98', 'tokens/total': 3509232, 'tokens/trainable': 53051, 'epoch': '0.4531'} |
|
23%|██▎ | 116/512 [12:54<43:44, 6.63s/it]
23%|██▎ | 117/512 [13:00<43:36, 6.62s/it]
{'loss': '0.006229', 'grad_norm': '0.3186', 'learning_rate': '9.247e-05', 'ppl': '1.006', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '35.85', 'tokens/total': 3539680, 'tokens/trainable': 53510, 'epoch': '0.457'} |
|
23%|██▎ | 117/512 [13:00<43:36, 6.62s/it]
23%|██▎ | 118/512 [13:07<43:32, 6.63s/it]
{'loss': '0.01025', 'grad_norm': '0.4676', 'learning_rate': '9.23e-05', 'ppl': '1.01', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '33.24', 'tokens/total': 3570320, 'tokens/trainable': 53966, 'epoch': '0.4609'} |
|
23%|██▎ | 118/512 [13:07<43:32, 6.63s/it]
23%|██▎ | 119/512 [13:14<43:18, 6.61s/it]
{'loss': '0.001145', 'grad_norm': '0.1801', 'learning_rate': '9.214e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.79', 'memory/max_allocated (GiB)': '33.79', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '35.02', 'tokens/total': 3600384, 'tokens/trainable': 54399, 'epoch': '0.4648'} |
|
23%|██▎ | 119/512 [13:14<43:18, 6.61s/it]
23%|██▎ | 120/512 [13:20<43:13, 6.62s/it]
{'loss': '0.01829', 'grad_norm': '0.4261', 'learning_rate': '9.198e-05', 'ppl': '1.018', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '36.98', 'tokens/total': 3630672, 'tokens/trainable': 54875, 'epoch': '0.4688'} |
|
23%|██▎ | 120/512 [13:20<43:13, 6.62s/it]
24%|██▎ | 121/512 [13:27<43:08, 6.62s/it]
{'loss': '0.00055', 'grad_norm': '0.0688', 'learning_rate': '9.181e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '32.87', 'tokens/total': 3660992, 'tokens/trainable': 55294, 'epoch': '0.4727'} |
|
24%|██▎ | 121/512 [13:27<43:08, 6.62s/it]
24%|██▍ | 122/512 [13:33<43:01, 6.62s/it]
{'loss': '0.009492', 'grad_norm': '0.333', 'learning_rate': '9.164e-05', 'ppl': '1.01', 'memory/max_active (GiB)': '33.77', 'memory/max_allocated (GiB)': '33.77', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '31.91', 'tokens/total': 3691248, 'tokens/trainable': 55734, 'epoch': '0.4766'} |
|
24%|██▍ | 122/512 [13:33<43:01, 6.62s/it]
24%|██▍ | 123/512 [13:40<42:51, 6.61s/it]
{'loss': '0.01347', 'grad_norm': '0.7759', 'learning_rate': '9.147e-05', 'ppl': '1.014', 'memory/max_active (GiB)': '33.73', 'memory/max_allocated (GiB)': '33.73', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '33.68', 'tokens/total': 3721248, 'tokens/trainable': 56178, 'epoch': '0.4805'} |
|
24%|██▍ | 123/512 [13:40<42:51, 6.61s/it]
24%|██▍ | 124/512 [13:47<42:45, 6.61s/it]
{'loss': '0.01168', 'grad_norm': '0.5912', 'learning_rate': '9.13e-05', 'ppl': '1.012', 'memory/max_active (GiB)': '33.77', 'memory/max_allocated (GiB)': '33.77', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '33.42', 'tokens/total': 3751440, 'tokens/trainable': 56623, 'epoch': '0.4844'} |
|
24%|██▍ | 124/512 [13:47<42:45, 6.61s/it]
24%|██▍ | 125/512 [13:53<42:43, 6.63s/it]
{'loss': '0.008603', 'grad_norm': '0.5725', 'learning_rate': '9.113e-05', 'ppl': '1.009', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '37.12', 'tokens/total': 3781696, 'tokens/trainable': 57115, 'epoch': '0.4883'} |
|
24%|██▍ | 125/512 [13:53<42:43, 6.63s/it]
25%|██▍ | 126/512 [14:00<42:33, 6.61s/it]
{'loss': '0.01081', 'grad_norm': '0.4081', 'learning_rate': '9.096e-05', 'ppl': '1.011', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '33.25', 'tokens/total': 3812096, 'tokens/trainable': 57561, 'epoch': '0.4922'} |
|
25%|██▍ | 126/512 [14:00<42:33, 6.61s/it]
25%|██▍ | 127/512 [14:07<42:32, 6.63s/it]
{'loss': '0.02943', 'grad_norm': '0.6532', 'learning_rate': '9.078e-05', 'ppl': '1.03', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '35.26', 'tokens/total': 3842720, 'tokens/trainable': 58039, 'epoch': '0.4961'} |
|
25%|██▍ | 127/512 [14:07<42:32, 6.63s/it]
25%|██▌ | 128/512 [14:13<41:25, 6.47s/it]
{'loss': '0.00902', 'grad_norm': '0.2376', 'learning_rate': '9.06e-05', 'ppl': '1.009', 'memory/max_active (GiB)': '33.39', 'memory/max_allocated (GiB)': '33.39', 'memory/device_reserved (GiB)': '35.99', 'tokens/train_per_sec_per_gpu': '35.7', 'tokens/total': 3871072, 'tokens/trainable': 58488, 'epoch': '0.5'} |
|
25%|██▌ | 128/512 [14:13<41:25, 6.47s/it][2026-08-18 14:45:56,343] [INFO] [axolotl.core.trainers.base._save:828] [PID:12035] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-128 |
|
25%|██▌ | 129/512 [14:21<44:39, 7.00s/it]
{'loss': '0.01388', 'grad_norm': '0.5178', 'learning_rate': '9.043e-05', 'ppl': '1.014', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '36.01', 'tokens/train_per_sec_per_gpu': '33.07', 'tokens/total': 3901504, 'tokens/trainable': 58938, 'epoch': '0.5039'} |
|
25%|██▌ | 129/512 [14:21<44:39, 7.00s/it]
25%|██▌ | 130/512 [14:28<43:52, 6.89s/it]
{'loss': '0.02103', 'grad_norm': '0.2801', 'learning_rate': '9.025e-05', 'ppl': '1.021', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '36.01', 'tokens/train_per_sec_per_gpu': '37.3', 'tokens/total': 3931968, 'tokens/trainable': 59406, 'epoch': '0.5078'} |
|
25%|██▌ | 130/512 [14:28<43:52, 6.89s/it]
26%|██▌ | 131/512 [14:34<43:13, 6.81s/it]
{'loss': '0.02409', 'grad_norm': '0.4327', 'learning_rate': '9.007e-05', 'ppl': '1.024', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '32.69', 'tokens/total': 3962048, 'tokens/trainable': 59842, 'epoch': '0.5117'} |
|
26%|██▌ | 131/512 [14:34<43:13, 6.81s/it]
26%|██▌ | 132/512 [14:41<42:48, 6.76s/it]
{'loss': '0.00735', 'grad_norm': '0.1669', 'learning_rate': '8.988e-05', 'ppl': '1.007', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '35.96', 'tokens/total': 3992512, 'tokens/trainable': 60316, 'epoch': '0.5156'} |
|
26%|██▌ | 132/512 [14:41<42:48, 6.76s/it]
26%|██▌ | 133/512 [14:47<42:28, 6.72s/it]
{'loss': '0.0138', 'grad_norm': '0.2077', 'learning_rate': '8.97e-05', 'ppl': '1.014', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '35.84', 'tokens/total': 4022864, 'tokens/trainable': 60809, 'epoch': '0.5195'} |
|
26%|██▌ | 133/512 [14:47<42:28, 6.72s/it]
26%|██▌ | 134/512 [14:54<41:14, 6.55s/it]
{'loss': '0.0167', 'grad_norm': '0.215', 'learning_rate': '8.951e-05', 'ppl': '1.017', 'memory/max_active (GiB)': '33.39', 'memory/max_allocated (GiB)': '33.39', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '32.14', 'tokens/total': 4051344, 'tokens/trainable': 61254, 'epoch': '0.5234'} |
|
26%|██▌ | 134/512 [14:54<41:14, 6.55s/it]
26%|██▋ | 135/512 [15:00<41:19, 6.58s/it]
{'loss': '0.01636', 'grad_norm': '0.3072', 'learning_rate': '8.933e-05', 'ppl': '1.016', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '36.41', 'tokens/total': 4081728, 'tokens/trainable': 61725, 'epoch': '0.5273'} |
|
26%|██▋ | 135/512 [15:00<41:19, 6.58s/it]
27%|██▋ | 136/512 [15:07<41:12, 6.57s/it]
{'loss': '0.0143', 'grad_norm': '0.2396', 'learning_rate': '8.914e-05', 'ppl': '1.014', 'memory/max_active (GiB)': '33.7', 'memory/max_allocated (GiB)': '33.7', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '38.67', 'tokens/total': 4111904, 'tokens/trainable': 62200, 'epoch': '0.5312'} |
|
27%|██▋ | 136/512 [15:07<41:12, 6.57s/it]
27%|██▋ | 137/512 [15:13<41:12, 6.59s/it]
{'loss': '0.01103', 'grad_norm': '0.1935', 'learning_rate': '8.895e-05', 'ppl': '1.011', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '36.9', 'tokens/total': 4142208, 'tokens/trainable': 62698, 'epoch': '0.5352'} |
|
27%|██▋ | 137/512 [15:13<41:12, 6.59s/it]
27%|██▋ | 138/512 [15:20<41:09, 6.60s/it]
{'loss': '0.02042', 'grad_norm': '0.3076', 'learning_rate': '8.876e-05', 'ppl': '1.021', 'memory/max_active (GiB)': '33.79', 'memory/max_allocated (GiB)': '33.79', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '37.02', 'tokens/total': 4172512, 'tokens/trainable': 63158, 'epoch': '0.5391'} |
|
27%|██▋ | 138/512 [15:20<41:09, 6.60s/it]
27%|██▋ | 139/512 [15:27<41:07, 6.62s/it]
{'loss': '0.007682', 'grad_norm': '0.2301', 'learning_rate': '8.856e-05', 'ppl': '1.008', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '36.87', 'tokens/total': 4202944, 'tokens/trainable': 63638, 'epoch': '0.543'} |
|
27%|██▋ | 139/512 [15:27<41:07, 6.62s/it]
27%|██▋ | 140/512 [15:33<40:02, 6.46s/it]
{'loss': '0.01174', 'grad_norm': '0.2613', 'learning_rate': '8.837e-05', 'ppl': '1.012', 'memory/max_active (GiB)': '33.42', 'memory/max_allocated (GiB)': '33.42', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '33.35', 'tokens/total': 4231264, 'tokens/trainable': 64055, 'epoch': '0.5469'} |
|
27%|██▋ | 140/512 [15:33<40:02, 6.46s/it]
28%|██▊ | 141/512 [15:39<40:19, 6.52s/it]
{'loss': '0.008032', 'grad_norm': '0.2588', 'learning_rate': '8.817e-05', 'ppl': '1.008', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '35.41', 'tokens/total': 4261488, 'tokens/trainable': 64512, 'epoch': '0.5508'} |
|
28%|██▊ | 141/512 [15:39<40:19, 6.52s/it]
28%|██▊ | 142/512 [15:46<40:26, 6.56s/it]
{'loss': '0.002808', 'grad_norm': '0.1194', 'learning_rate': '8.798e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '34.31', 'tokens/total': 4292032, 'tokens/trainable': 64950, 'epoch': '0.5547'} |
|
28%|██▊ | 142/512 [15:46<40:26, 6.56s/it]
28%|██▊ | 143/512 [15:53<40:22, 6.56s/it]
{'loss': '0.004209', 'grad_norm': '0.1714', 'learning_rate': '8.778e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '37.89', 'tokens/train_per_sec_per_gpu': '38.15', 'tokens/total': 4322032, 'tokens/trainable': 65429, 'epoch': '0.5586'} |
|
28%|██▊ | 143/512 [15:53<40:22, 6.56s/it]
28%|██▊ | 144/512 [15:59<40:19, 6.57s/it]
{'loss': '0.003017', 'grad_norm': '0.1641', 'learning_rate': '8.758e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '37.89', 'tokens/train_per_sec_per_gpu': '37.17', 'tokens/total': 4352128, 'tokens/trainable': 65895, 'epoch': '0.5625'} |
|
28%|██▊ | 144/512 [15:59<40:19, 6.57s/it]
28%|██▊ | 145/512 [16:06<40:15, 6.58s/it]
{'loss': '0.001614', 'grad_norm': '0.1339', 'learning_rate': '8.738e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.79', 'memory/max_allocated (GiB)': '33.79', 'memory/device_reserved (GiB)': '37.89', 'tokens/train_per_sec_per_gpu': '31.99', 'tokens/total': 4382368, 'tokens/trainable': 66327, 'epoch': '0.5664'} |
|
28%|██▊ | 145/512 [16:06<40:15, 6.58s/it]
29%|██▊ | 146/512 [16:13<40:14, 6.60s/it]
{'loss': '0.008623', 'grad_norm': '0.566', 'learning_rate': '8.718e-05', 'ppl': '1.009', 'memory/max_active (GiB)': '33.74', 'memory/max_allocated (GiB)': '33.74', 'memory/device_reserved (GiB)': '37.89', 'tokens/train_per_sec_per_gpu': '31.94', 'tokens/total': 4412640, 'tokens/trainable': 66778, 'epoch': '0.5703'} |
|
29%|██▊ | 146/512 [16:13<40:14, 6.60s/it]
29%|██▊ | 147/512 [16:19<40:10, 6.60s/it]
{'loss': '0.008356', 'grad_norm': '0.4248', 'learning_rate': '8.697e-05', 'ppl': '1.008', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '37.89', 'tokens/train_per_sec_per_gpu': '35.67', 'tokens/total': 4442896, 'tokens/trainable': 67192, 'epoch': '0.5742'} |
|
29%|██▊ | 147/512 [16:19<40:10, 6.60s/it]
29%|██▉ | 148/512 [16:26<40:12, 6.63s/it]
{'loss': '0.01054', 'grad_norm': '0.401', 'learning_rate': '8.677e-05', 'ppl': '1.011', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '37.89', 'tokens/train_per_sec_per_gpu': '35.79', 'tokens/total': 4473248, 'tokens/trainable': 67677, 'epoch': '0.5781'} |
|
29%|██▉ | 148/512 [16:26<40:12, 6.63s/it]
29%|██▉ | 149/512 [16:32<40:05, 6.63s/it]
{'loss': '0.002294', 'grad_norm': '0.179', 'learning_rate': '8.656e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '37.89', 'tokens/train_per_sec_per_gpu': '34.25', 'tokens/total': 4503664, 'tokens/trainable': 68121, 'epoch': '0.582'} |
|
29%|██▉ | 149/512 [16:32<40:05, 6.63s/it]
29%|██▉ | 150/512 [16:39<40:00, 6.63s/it]
{'loss': '0.002104', 'grad_norm': '0.49', 'learning_rate': '8.635e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '37.89', 'tokens/train_per_sec_per_gpu': '33.44', 'tokens/total': 4533760, 'tokens/trainable': 68589, 'epoch': '0.5859'} |
|
29%|██▉ | 150/512 [16:39<40:00, 6.63s/it]
29%|██▉ | 151/512 [16:46<40:02, 6.66s/it]
{'loss': '0.02125', 'grad_norm': '0.6358', 'learning_rate': '8.615e-05', 'ppl': '1.021', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '37.89', 'tokens/train_per_sec_per_gpu': '30.28', 'tokens/total': 4563872, 'tokens/trainable': 69055, 'epoch': '0.5898'} |
|
29%|██▉ | 151/512 [16:46<40:02, 6.66s/it]
30%|██▉ | 152/512 [16:52<39:55, 6.66s/it]
{'loss': '0.01092', 'grad_norm': '0.8154', 'learning_rate': '8.594e-05', 'ppl': '1.011', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '37.89', 'tokens/train_per_sec_per_gpu': '35.77', 'tokens/total': 4594384, 'tokens/trainable': 69501, 'epoch': '0.5938'} |
|
30%|██▉ | 152/512 [16:52<39:55, 6.66s/it]
30%|██▉ | 153/512 [16:59<39:52, 6.66s/it]
{'loss': '0.009749', 'grad_norm': '0.5218', 'learning_rate': '8.572e-05', 'ppl': '1.01', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '37.89', 'tokens/train_per_sec_per_gpu': '34.26', 'tokens/total': 4624816, 'tokens/trainable': 69964, 'epoch': '0.5977'} |
|
30%|██▉ | 153/512 [16:59<39:52, 6.66s/it]
30%|███ | 154/512 [17:06<39:42, 6.65s/it]
{'loss': '0.003197', 'grad_norm': '0.2303', 'learning_rate': '8.551e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '37.89', 'tokens/train_per_sec_per_gpu': '37.11', 'tokens/total': 4655136, 'tokens/trainable': 70431, 'epoch': '0.6016'} |
|
30%|███ | 154/512 [17:06<39:42, 6.65s/it]
30%|███ | 155/512 [17:12<38:37, 6.49s/it]
{'loss': '0.0492', 'grad_norm': '1.329', 'learning_rate': '8.53e-05', 'ppl': '1.05', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '37.89', 'tokens/train_per_sec_per_gpu': '37.67', 'tokens/total': 4683488, 'tokens/trainable': 70875, 'epoch': '0.6055'} |
|
30%|███ | 155/512 [17:12<38:37, 6.49s/it]
30%|███ | 156/512 [17:19<38:47, 6.54s/it]
{'loss': '0.02088', 'grad_norm': '0.6996', 'learning_rate': '8.508e-05', 'ppl': '1.021', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '37.89', 'tokens/train_per_sec_per_gpu': '34.31', 'tokens/total': 4714032, 'tokens/trainable': 71333, 'epoch': '0.6094'} |
|
30%|███ | 156/512 [17:19<38:47, 6.54s/it]
31%|███ | 157/512 [17:25<38:50, 6.56s/it]
{'loss': '0.003256', 'grad_norm': '0.2432', 'learning_rate': '8.487e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.79', 'memory/max_allocated (GiB)': '33.79', 'memory/device_reserved (GiB)': '37.89', 'tokens/train_per_sec_per_gpu': '35.46', 'tokens/total': 4744192, 'tokens/trainable': 71795, 'epoch': '0.6133'} |
|
31%|███ | 157/512 [17:25<38:50, 6.56s/it]
31%|███ | 158/512 [17:32<38:54, 6.59s/it]
{'loss': '0.01598', 'grad_norm': '0.5778', 'learning_rate': '8.465e-05', 'ppl': '1.016', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '37.89', 'tokens/train_per_sec_per_gpu': '32.59', 'tokens/total': 4774592, 'tokens/trainable': 72231, 'epoch': '0.6172'} |
|
31%|███ | 158/512 [17:32<38:54, 6.59s/it]
31%|███ | 159/512 [17:38<38:45, 6.59s/it]
{'loss': '0.01524', 'grad_norm': '0.374', 'learning_rate': '8.443e-05', 'ppl': '1.015', 'memory/max_active (GiB)': '33.75', 'memory/max_allocated (GiB)': '33.75', 'memory/device_reserved (GiB)': '37.89', 'tokens/train_per_sec_per_gpu': '30.89', 'tokens/total': 4804672, 'tokens/trainable': 72660, 'epoch': '0.6211'} |
|
31%|███ | 159/512 [17:38<38:45, 6.59s/it]
31%|███▏ | 160/512 [17:45<38:44, 6.60s/it]
{'loss': '0.02623', 'grad_norm': '0.5424', 'learning_rate': '8.421e-05', 'ppl': '1.027', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '37.89', 'tokens/train_per_sec_per_gpu': '31.94', 'tokens/total': 4835056, 'tokens/trainable': 73115, 'epoch': '0.625'} |
|
31%|███▏ | 160/512 [17:45<38:44, 6.60s/it][2026-08-18 14:49:28,679] [INFO] [axolotl.core.trainers.base._save:828] [PID:12035] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-160 |
|
31%|███▏ | 161/512 [17:54<42:15, 7.22s/it]
{'loss': '0.009905', 'grad_norm': '0.2063', 'learning_rate': '8.399e-05', 'ppl': '1.01', 'memory/max_active (GiB)': '33.75', 'memory/max_allocated (GiB)': '33.75', 'memory/device_reserved (GiB)': '37.89', 'tokens/train_per_sec_per_gpu': '31.52', 'tokens/total': 4865296, 'tokens/trainable': 73536, 'epoch': '0.6289'} |
|
31%|███▏ | 161/512 [17:54<42:15, 7.22s/it]
32%|███▏ | 162/512 [18:00<41:09, 7.06s/it]
{'loss': '0.01187', 'grad_norm': '0.2795', 'learning_rate': '8.376e-05', 'ppl': '1.012', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '35.81', 'tokens/train_per_sec_per_gpu': '33.04', 'tokens/total': 4895872, 'tokens/trainable': 73985, 'epoch': '0.6328'} |
|
32%|███▏ | 162/512 [18:00<41:09, 7.06s/it]
32%|███▏ | 163/512 [18:07<40:18, 6.93s/it]
{'loss': '0.01288', 'grad_norm': '0.2867', 'learning_rate': '8.354e-05', 'ppl': '1.013', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '35.81', 'tokens/train_per_sec_per_gpu': '35.7', 'tokens/total': 4926192, 'tokens/trainable': 74449, 'epoch': '0.6367'} |
|
32%|███▏ | 163/512 [18:07<40:18, 6.93s/it]
32%|███▏ | 164/512 [18:14<39:56, 6.89s/it]
{'loss': '0.01033', 'grad_norm': '0.1896', 'learning_rate': '8.332e-05', 'ppl': '1.01', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.81', 'tokens/train_per_sec_per_gpu': '33.78', 'tokens/total': 4956784, 'tokens/trainable': 74900, 'epoch': '0.6406'} |
|
32%|███▏ | 164/512 [18:14<39:56, 6.89s/it]
32%|███▏ | 165/512 [18:20<39:19, 6.80s/it]
{'loss': '0.008086', 'grad_norm': '0.2795', 'learning_rate': '8.309e-05', 'ppl': '1.008', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '30.91', 'tokens/total': 4987120, 'tokens/trainable': 75324, 'epoch': '0.6445'} |
|
32%|███▏ | 165/512 [18:20<39:19, 6.80s/it]
32%|███▏ | 166/512 [18:27<38:50, 6.74s/it]
{'loss': '0.002372', 'grad_norm': '0.2087', 'learning_rate': '8.286e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.75', 'memory/max_allocated (GiB)': '33.75', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '32.21', 'tokens/total': 5017424, 'tokens/trainable': 75765, 'epoch': '0.6484'} |
|
32%|███▏ | 166/512 [18:27<38:50, 6.74s/it]
33%|███▎ | 167/512 [18:34<38:44, 6.74s/it]
{'loss': '0.006408', 'grad_norm': '0.1626', 'learning_rate': '8.263e-05', 'ppl': '1.006', 'memory/max_active (GiB)': '33.99', 'memory/max_allocated (GiB)': '33.99', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '31.6', 'tokens/total': 5048096, 'tokens/trainable': 76226, 'epoch': '0.6523'} |
|
33%|███▎ | 167/512 [18:34<38:44, 6.74s/it]
33%|███▎ | 168/512 [18:40<38:32, 6.72s/it]
{'loss': '0.01599', 'grad_norm': '0.2421', 'learning_rate': '8.24e-05', 'ppl': '1.016', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '33.52', 'tokens/total': 5078608, 'tokens/trainable': 76677, 'epoch': '0.6562'} |
|
33%|███▎ | 168/512 [18:40<38:32, 6.72s/it]
33%|███▎ | 169/512 [18:47<38:16, 6.70s/it]
{'loss': '0.001437', 'grad_norm': '0.09416', 'learning_rate': '8.217e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '34.52', 'tokens/total': 5108944, 'tokens/trainable': 77109, 'epoch': '0.6602'} |
|
33%|███▎ | 169/512 [18:47<38:16, 6.70s/it]
33%|███▎ | 170/512 [18:54<38:08, 6.69s/it]
{'loss': '0.01865', 'grad_norm': '0.351', 'learning_rate': '8.194e-05', 'ppl': '1.019', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '36.01', 'tokens/train_per_sec_per_gpu': '34.43', 'tokens/total': 5139344, 'tokens/trainable': 77558, 'epoch': '0.6641'} |
|
33%|███▎ | 170/512 [18:54<38:08, 6.69s/it]
33%|███▎ | 171/512 [19:00<37:53, 6.67s/it]
{'loss': '0.0173', 'grad_norm': '0.5683', 'learning_rate': '8.171e-05', 'ppl': '1.017', 'memory/max_active (GiB)': '33.79', 'memory/max_allocated (GiB)': '33.79', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '33.58', 'tokens/total': 5169600, 'tokens/trainable': 78018, 'epoch': '0.668'} |
|
33%|███▎ | 171/512 [19:00<37:53, 6.67s/it]
34%|███▎ | 172/512 [19:07<37:47, 6.67s/it]
{'loss': '0.005543', 'grad_norm': '0.4383', 'learning_rate': '8.147e-05', 'ppl': '1.006', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '30.15', 'tokens/total': 5200032, 'tokens/trainable': 78472, 'epoch': '0.6719'} |
|
34%|███▎ | 172/512 [19:07<37:47, 6.67s/it]
34%|███▍ | 173/512 [19:14<37:32, 6.64s/it]
{'loss': '0.00764', 'grad_norm': '0.3213', 'learning_rate': '8.124e-05', 'ppl': '1.008', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '33.42', 'tokens/total': 5230464, 'tokens/trainable': 78919, 'epoch': '0.6758'} |
|
34%|███▍ | 173/512 [19:14<37:32, 6.64s/it]
34%|███▍ | 174/512 [19:20<37:08, 6.59s/it]
{'loss': '0.005054', 'grad_norm': '0.2374', 'learning_rate': '8.1e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.62', 'memory/max_allocated (GiB)': '33.62', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '30.4', 'tokens/total': 5260192, 'tokens/trainable': 79336, 'epoch': '0.6797'} |
|
34%|███▍ | 174/512 [19:20<37:08, 6.59s/it]
34%|███▍ | 175/512 [19:27<37:05, 6.60s/it]
{'loss': '0.001754', 'grad_norm': '0.1102', 'learning_rate': '8.076e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '32.91', 'tokens/total': 5290480, 'tokens/trainable': 79806, 'epoch': '0.6836'} |
|
34%|███▍ | 175/512 [19:27<37:05, 6.60s/it]
34%|███▍ | 176/512 [19:33<37:09, 6.63s/it]
{'loss': '0.009171', 'grad_norm': '0.5749', 'learning_rate': '8.053e-05', 'ppl': '1.009', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '36.4', 'tokens/total': 5320992, 'tokens/trainable': 80284, 'epoch': '0.6875'} |
|
34%|███▍ | 176/512 [19:33<37:09, 6.63s/it]
35%|███▍ | 177/512 [19:40<37:09, 6.66s/it]
{'loss': '0.001649', 'grad_norm': '0.09885', 'learning_rate': '8.029e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '32.5', 'tokens/total': 5351328, 'tokens/trainable': 80731, 'epoch': '0.6914'} |
|
35%|███▍ | 177/512 [19:40<37:09, 6.66s/it]
35%|███▍ | 178/512 [19:47<37:07, 6.67s/it]
{'loss': '0.004666', 'grad_norm': '0.2898', 'learning_rate': '8.005e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '34.93', 'tokens/total': 5381536, 'tokens/trainable': 81199, 'epoch': '0.6953'} |
|
35%|███▍ | 178/512 [19:47<37:07, 6.67s/it]
35%|███▍ | 179/512 [19:53<36:54, 6.65s/it]
{'loss': '0.001716', 'grad_norm': '0.2229', 'learning_rate': '7.98e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '33.18', 'tokens/total': 5411536, 'tokens/trainable': 81646, 'epoch': '0.6992'} |
|
35%|███▍ | 179/512 [19:53<36:54, 6.65s/it]
35%|███▌ | 180/512 [20:00<36:51, 6.66s/it]
{'loss': '0.0004056', 'grad_norm': '0.01561', 'learning_rate': '7.956e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '34.69', 'tokens/total': 5441712, 'tokens/trainable': 82119, 'epoch': '0.7031'} |
|
35%|███▌ | 180/512 [20:00<36:51, 6.66s/it]
35%|███▌ | 181/512 [20:07<36:48, 6.67s/it]
{'loss': '0.001177', 'grad_norm': '0.06141', 'learning_rate': '7.932e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '39.13', 'tokens/total': 5471968, 'tokens/trainable': 82636, 'epoch': '0.707'} |
|
35%|███▌ | 181/512 [20:07<36:48, 6.67s/it]
36%|███▌ | 182/512 [20:13<36:37, 6.66s/it]
{'loss': '0.004583', 'grad_norm': '0.3485', 'learning_rate': '7.907e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '34.4', 'tokens/total': 5502320, 'tokens/trainable': 83116, 'epoch': '0.7109'} |
|
36%|███▌ | 182/512 [20:13<36:37, 6.66s/it]
36%|███▌ | 183/512 [20:20<36:27, 6.65s/it]
{'loss': '0.009528', 'grad_norm': '0.3753', 'learning_rate': '7.883e-05', 'ppl': '1.01', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '33.23', 'tokens/total': 5532512, 'tokens/trainable': 83563, 'epoch': '0.7148'} |
|
36%|███▌ | 183/512 [20:20<36:27, 6.65s/it]
36%|███▌ | 184/512 [20:27<36:17, 6.64s/it]
{'loss': '0.0001315', 'grad_norm': '0.005008', 'learning_rate': '7.858e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '35.95', 'tokens/total': 5562992, 'tokens/trainable': 84069, 'epoch': '0.7188'} |
|
36%|███▌ | 184/512 [20:27<36:17, 6.64s/it]
36%|███▌ | 185/512 [20:33<36:11, 6.64s/it]
{'loss': '0.0007049', 'grad_norm': '0.04002', 'learning_rate': '7.833e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '34.77', 'tokens/total': 5593200, 'tokens/trainable': 84544, 'epoch': '0.7227'} |
|
36%|███▌ | 185/512 [20:33<36:11, 6.64s/it]
36%|███▋ | 186/512 [20:40<36:07, 6.65s/it]
{'loss': '0.002376', 'grad_norm': '0.2496', 'learning_rate': '7.808e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '33.75', 'tokens/total': 5623568, 'tokens/trainable': 85005, 'epoch': '0.7266'} |
|
36%|███▋ | 186/512 [20:40<36:07, 6.65s/it]
37%|███▋ | 187/512 [20:47<35:59, 6.64s/it]
{'loss': '0.0004841', 'grad_norm': '0.03484', 'learning_rate': '7.783e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '32.14', 'tokens/total': 5653856, 'tokens/trainable': 85445, 'epoch': '0.7305'} |
|
37%|███▋ | 187/512 [20:47<35:59, 6.64s/it]
37%|███▋ | 188/512 [20:53<35:54, 6.65s/it]
{'loss': '0.02416', 'grad_norm': '0.8146', 'learning_rate': '7.758e-05', 'ppl': '1.024', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '34.38', 'tokens/total': 5684112, 'tokens/trainable': 85935, 'epoch': '0.7344'} |
|
37%|███▋ | 188/512 [20:53<35:54, 6.65s/it]
37%|███▋ | 189/512 [21:00<35:48, 6.65s/it]
{'loss': '0.01051', 'grad_norm': '0.5074', 'learning_rate': '7.733e-05', 'ppl': '1.011', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '36.63', 'tokens/total': 5714400, 'tokens/trainable': 86450, 'epoch': '0.7383'} |
|
37%|███▋ | 189/512 [21:00<35:48, 6.65s/it]
37%|███▋ | 190/512 [21:07<35:40, 6.65s/it]
{'loss': '0.01207', 'grad_norm': '0.4441', 'learning_rate': '7.708e-05', 'ppl': '1.012', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '34.21', 'tokens/total': 5744992, 'tokens/trainable': 86921, 'epoch': '0.7422'} |
|
37%|███▋ | 190/512 [21:07<35:40, 6.65s/it]
37%|███▋ | 191/512 [21:13<35:37, 6.66s/it]
{'loss': '0.005623', 'grad_norm': '0.2527', 'learning_rate': '7.683e-05', 'ppl': '1.006', 'memory/max_active (GiB)': '33.99', 'memory/max_allocated (GiB)': '33.99', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '36.5', 'tokens/total': 5775568, 'tokens/trainable': 87376, 'epoch': '0.7461'} |
|
37%|███▋ | 191/512 [21:13<35:37, 6.66s/it]
38%|███▊ | 192/512 [21:20<35:26, 6.64s/it]
{'loss': '0.01618', 'grad_norm': '0.4064', 'learning_rate': '7.657e-05', 'ppl': '1.016', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '31.93', 'tokens/total': 5806000, 'tokens/trainable': 87806, 'epoch': '0.75'} |
|
38%|███▊ | 192/512 [21:20<35:26, 6.64s/it][2026-08-18 14:53:03,511] [INFO] [axolotl.core.trainers.base._save:828] [PID:12035] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-192 |
|
38%|███▊ | 193/512 [21:28<37:53, 7.13s/it]
{'loss': '0.008485', 'grad_norm': '0.263', 'learning_rate': '7.632e-05', 'ppl': '1.009', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '36.02', 'tokens/train_per_sec_per_gpu': '35.5', 'tokens/total': 5836432, 'tokens/trainable': 88289, 'epoch': '0.7539'} |
|
38%|███▊ | 193/512 [21:28<37:53, 7.13s/it]
38%|███▊ | 194/512 [21:35<37:02, 6.99s/it]
{'loss': '0.002483', 'grad_norm': '0.1066', 'learning_rate': '7.606e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '35.27', 'tokens/total': 5866752, 'tokens/trainable': 88760, 'epoch': '0.7578'} |
|
38%|███▊ | 194/512 [21:35<37:02, 6.99s/it]
38%|███▊ | 195/512 [21:41<35:27, 6.71s/it]
{'loss': '0.00554', 'grad_norm': '0.1543', 'learning_rate': '7.58e-05', 'ppl': '1.006', 'memory/max_active (GiB)': '33.38', 'memory/max_allocated (GiB)': '33.38', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '30.83', 'tokens/total': 5894976, 'tokens/trainable': 89193, 'epoch': '0.7617'} |
|
38%|███▊ | 195/512 [21:41<35:27, 6.71s/it]
38%|███▊ | 196/512 [21:47<35:13, 6.69s/it]
{'loss': '0.003556', 'grad_norm': '0.1539', 'learning_rate': '7.555e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '37.72', 'tokens/total': 5925376, 'tokens/trainable': 89655, 'epoch': '0.7656'} |
|
38%|███▊ | 196/512 [21:47<35:13, 6.69s/it]
38%|███▊ | 197/512 [21:54<34:14, 6.52s/it]
{'loss': '0.007582', 'grad_norm': '0.2556', 'learning_rate': '7.529e-05', 'ppl': '1.008', 'memory/max_active (GiB)': '33.34', 'memory/max_allocated (GiB)': '33.34', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '38.46', 'tokens/total': 5953696, 'tokens/trainable': 90103, 'epoch': '0.7695'} |
|
38%|███▊ | 197/512 [21:54<34:14, 6.52s/it]
39%|███▊ | 198/512 [22:00<34:20, 6.56s/it]
{'loss': '0.01378', 'grad_norm': '0.3198', 'learning_rate': '7.503e-05', 'ppl': '1.014', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '34.59', 'tokens/total': 5983952, 'tokens/trainable': 90564, 'epoch': '0.7734'} |
|
39%|███▊ | 198/512 [22:00<34:20, 6.56s/it]
39%|███▉ | 199/512 [22:07<34:34, 6.63s/it]
{'loss': '0.009123', 'grad_norm': '0.2523', 'learning_rate': '7.477e-05', 'ppl': '1.009', 'memory/max_active (GiB)': '34', 'memory/max_allocated (GiB)': '34', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '36.87', 'tokens/total': 6014704, 'tokens/trainable': 91043, 'epoch': '0.7773'} |
|
39%|███▉ | 199/512 [22:07<34:34, 6.63s/it]
39%|███▉ | 200/512 [22:14<34:33, 6.65s/it]
{'loss': '0.007986', 'grad_norm': '0.3038', 'learning_rate': '7.451e-05', 'ppl': '1.008', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '38.27', 'tokens/total': 6044960, 'tokens/trainable': 91519, 'epoch': '0.7812'} |
|
39%|███▉ | 200/512 [22:14<34:33, 6.65s/it]
39%|███▉ | 201/512 [22:20<34:25, 6.64s/it]
{'loss': '0.002435', 'grad_norm': '0.07847', 'learning_rate': '7.424e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.69', 'memory/max_allocated (GiB)': '33.69', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '34.43', 'tokens/total': 6074960, 'tokens/trainable': 91965, 'epoch': '0.7852'} |
|
39%|███▉ | 201/512 [22:20<34:25, 6.64s/it]
39%|███▉ | 202/512 [22:27<34:18, 6.64s/it]
{'loss': '0.00519', 'grad_norm': '0.1524', 'learning_rate': '7.398e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '36.89', 'tokens/total': 6105312, 'tokens/trainable': 92430, 'epoch': '0.7891'} |
|
39%|███▉ | 202/512 [22:27<34:18, 6.64s/it]
40%|███▉ | 203/512 [22:34<34:08, 6.63s/it]
{'loss': '0.02347', 'grad_norm': '0.3384', 'learning_rate': '7.372e-05', 'ppl': '1.024', 'memory/max_active (GiB)': '33.77', 'memory/max_allocated (GiB)': '33.77', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '37', 'tokens/total': 6135392, 'tokens/trainable': 92877, 'epoch': '0.793'} |
|
40%|███▉ | 203/512 [22:34<34:08, 6.63s/it]
40%|███▉ | 204/512 [22:40<34:00, 6.62s/it]
{'loss': '0.006684', 'grad_norm': '0.1986', 'learning_rate': '7.345e-05', 'ppl': '1.007', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '34.46', 'tokens/total': 6165936, 'tokens/trainable': 93350, 'epoch': '0.7969'} |
|
40%|███▉ | 204/512 [22:40<34:00, 6.62s/it]
40%|████ | 205/512 [22:47<33:47, 6.60s/it]
{'loss': '0.007712', 'grad_norm': '0.2053', 'learning_rate': '7.319e-05', 'ppl': '1.008', 'memory/max_active (GiB)': '33.76', 'memory/max_allocated (GiB)': '33.76', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '33.89', 'tokens/total': 6196128, 'tokens/trainable': 93774, 'epoch': '0.8008'} |
|
40%|████ | 205/512 [22:47<33:47, 6.60s/it]
40%|████ | 206/512 [22:53<33:43, 6.61s/it]
{'loss': '0.04438', 'grad_norm': '0.6116', 'learning_rate': '7.292e-05', 'ppl': '1.045', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '38.11', 'tokens/total': 6226288, 'tokens/trainable': 94253, 'epoch': '0.8047'} |
|
40%|████ | 206/512 [22:53<33:43, 6.61s/it]
40%|████ | 207/512 [23:00<33:38, 6.62s/it]
{'loss': '0.009884', 'grad_norm': '0.2218', 'learning_rate': '7.266e-05', 'ppl': '1.01', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '35.78', 'tokens/total': 6256448, 'tokens/trainable': 94743, 'epoch': '0.8086'} |
|
40%|████ | 207/512 [23:00<33:38, 6.62s/it]
41%|████ | 208/512 [23:07<33:31, 6.62s/it]
{'loss': '0.004908', 'grad_norm': '0.1375', 'learning_rate': '7.239e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '34.16', 'tokens/total': 6286736, 'tokens/trainable': 95228, 'epoch': '0.8125'} |
|
41%|████ | 208/512 [23:07<33:31, 6.62s/it]
41%|████ | 209/512 [23:13<33:20, 6.60s/it]
{'loss': '0.004732', 'grad_norm': '0.1199', 'learning_rate': '7.212e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '29.53', 'tokens/total': 6317376, 'tokens/trainable': 95629, 'epoch': '0.8164'} |
|
41%|████ | 209/512 [23:13<33:20, 6.60s/it]
41%|████ | 210/512 [23:20<33:16, 6.61s/it]
{'loss': '0.009559', 'grad_norm': '0.1896', 'learning_rate': '7.185e-05', 'ppl': '1.01', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '35.79', 'tokens/total': 6347632, 'tokens/trainable': 96086, 'epoch': '0.8203'} |
|
41%|████ | 210/512 [23:20<33:16, 6.61s/it]
41%|████ | 211/512 [23:26<33:06, 6.60s/it]
{'loss': '0.0133', 'grad_norm': '0.2574', 'learning_rate': '7.158e-05', 'ppl': '1.013', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '34.56', 'tokens/total': 6377616, 'tokens/trainable': 96507, 'epoch': '0.8242'} |
|
41%|████ | 211/512 [23:26<33:06, 6.60s/it]
41%|████▏ | 212/512 [23:33<33:01, 6.61s/it]
{'loss': '0.007867', 'grad_norm': '0.2428', 'learning_rate': '7.131e-05', 'ppl': '1.008', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '32.92', 'tokens/total': 6408128, 'tokens/trainable': 96931, 'epoch': '0.8281'} |
|
41%|████▏ | 212/512 [23:33<33:01, 6.61s/it]
42%|████▏ | 213/512 [23:40<32:56, 6.61s/it]
{'loss': '0.007124', 'grad_norm': '0.1859', 'learning_rate': '7.104e-05', 'ppl': '1.007', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '33.11', 'tokens/total': 6438560, 'tokens/trainable': 97369, 'epoch': '0.832'} |
|
42%|████▏ | 213/512 [23:40<32:56, 6.61s/it]
42%|████▏ | 214/512 [23:46<32:52, 6.62s/it]
{'loss': '0.008005', 'grad_norm': '0.382', 'learning_rate': '7.077e-05', 'ppl': '1.008', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '36.86', 'tokens/total': 6468832, 'tokens/trainable': 97858, 'epoch': '0.8359'} |
|
42%|████▏ | 214/512 [23:46<32:52, 6.62s/it]
42%|████▏ | 215/512 [23:53<32:40, 6.60s/it]
{'loss': '0.004251', 'grad_norm': '0.1681', 'learning_rate': '7.05e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '37.97', 'tokens/total': 6498992, 'tokens/trainable': 98308, 'epoch': '0.8398'} |
|
42%|████▏ | 215/512 [23:53<32:40, 6.60s/it]
42%|████▏ | 216/512 [23:59<32:33, 6.60s/it]
{'loss': '0.0003424', 'grad_norm': '0.01197', 'learning_rate': '7.022e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '36.89', 'tokens/total': 6529344, 'tokens/trainable': 98781, 'epoch': '0.8438'} |
|
42%|████▏ | 216/512 [23:59<32:33, 6.60s/it]
42%|████▏ | 217/512 [24:06<32:30, 6.61s/it]
{'loss': '0.001047', 'grad_norm': '0.07292', 'learning_rate': '6.995e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '37.21', 'tokens/total': 6559696, 'tokens/trainable': 99266, 'epoch': '0.8477'} |
|
42%|████▏ | 217/512 [24:06<32:30, 6.61s/it]
43%|████▎ | 218/512 [24:12<31:41, 6.47s/it]
{'loss': '0.0004653', 'grad_norm': '0.02374', 'learning_rate': '6.968e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '39.98', 'tokens/total': 6588048, 'tokens/trainable': 99739, 'epoch': '0.8516'} |
|
43%|████▎ | 218/512 [24:12<31:41, 6.47s/it]
43%|████▎ | 219/512 [24:19<31:47, 6.51s/it]
{'loss': '0.0005198', 'grad_norm': '0.021', 'learning_rate': '6.94e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '32.08', 'tokens/total': 6618480, 'tokens/trainable': 100172, 'epoch': '0.8555'} |
|
43%|████▎ | 219/512 [24:19<31:47, 6.51s/it]
43%|████▎ | 220/512 [24:25<31:54, 6.56s/it]
{'loss': '0.000921', 'grad_norm': '0.06873', 'learning_rate': '6.913e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '37.96', 'tokens/total': 6648880, 'tokens/trainable': 100646, 'epoch': '0.8594'} |
|
43%|████▎ | 220/512 [24:25<31:54, 6.56s/it]
43%|████▎ | 221/512 [24:32<31:53, 6.57s/it]
{'loss': '0.001164', 'grad_norm': '0.1226', 'learning_rate': '6.885e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '32.05', 'tokens/total': 6679040, 'tokens/trainable': 101097, 'epoch': '0.8633'} |
|
43%|████▎ | 221/512 [24:32<31:53, 6.57s/it]
43%|████▎ | 222/512 [24:39<31:49, 6.59s/it]
{'loss': '0.004546', 'grad_norm': '0.2159', 'learning_rate': '6.857e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '38.27', 'tokens/total': 6709552, 'tokens/trainable': 101563, 'epoch': '0.8672'} |
|
43%|████▎ | 222/512 [24:39<31:49, 6.59s/it]
44%|████▎ | 223/512 [24:45<31:43, 6.59s/it]
{'loss': '0.0181', 'grad_norm': '0.8475', 'learning_rate': '6.83e-05', 'ppl': '1.018', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '31.91', 'tokens/total': 6739936, 'tokens/trainable': 102004, 'epoch': '0.8711'} |
|
44%|████▎ | 223/512 [24:45<31:43, 6.59s/it]
44%|████▍ | 224/512 [24:52<31:40, 6.60s/it]
{'loss': '0.0005447', 'grad_norm': '0.0483', 'learning_rate': '6.802e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '35.91', 'tokens/total': 6770144, 'tokens/trainable': 102452, 'epoch': '0.875'} |
|
44%|████▍ | 224/512 [24:52<31:40, 6.60s/it][2026-08-18 14:56:35,597] [INFO] [axolotl.core.trainers.base._save:828] [PID:12035] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-224 |
|
44%|████▍ | 225/512 [25:00<33:47, 7.06s/it]
{'loss': '0.007109', 'grad_norm': '0.6089', 'learning_rate': '6.774e-05', 'ppl': '1.007', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '29.54', 'tokens/total': 6800688, 'tokens/trainable': 102868, 'epoch': '0.8789'} |
|
44%|████▍ | 225/512 [25:00<33:47, 7.06s/it]
44%|████▍ | 226/512 [25:07<33:16, 6.98s/it]
{'loss': '0.004347', 'grad_norm': '0.3519', 'learning_rate': '6.746e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '35.17', 'tokens/train_per_sec_per_gpu': '37.74', 'tokens/total': 6831376, 'tokens/trainable': 103375, 'epoch': '0.8828'} |
|
44%|████▍ | 226/512 [25:07<33:16, 6.98s/it]
44%|████▍ | 227/512 [25:13<32:37, 6.87s/it]
{'loss': '0.01161', 'grad_norm': '0.6944', 'learning_rate': '6.718e-05', 'ppl': '1.012', 'memory/max_active (GiB)': '33.76', 'memory/max_allocated (GiB)': '33.76', 'memory/device_reserved (GiB)': '35.17', 'tokens/train_per_sec_per_gpu': '39.83', 'tokens/total': 6861392, 'tokens/trainable': 103868, 'epoch': '0.8867'} |
|
44%|████▍ | 227/512 [25:13<32:37, 6.87s/it]
45%|████▍ | 228/512 [25:20<32:09, 6.79s/it]
{'loss': '0.0006434', 'grad_norm': '0.06135', 'learning_rate': '6.69e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '35.86', 'tokens/train_per_sec_per_gpu': '36.72', 'tokens/total': 6891776, 'tokens/trainable': 104331, 'epoch': '0.8906'} |
|
45%|████▍ | 228/512 [25:20<32:09, 6.79s/it]
45%|████▍ | 229/512 [25:27<31:44, 6.73s/it]
{'loss': '0.001245', 'grad_norm': '0.1004', 'learning_rate': '6.662e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.69', 'memory/max_allocated (GiB)': '33.69', 'memory/device_reserved (GiB)': '35.86', 'tokens/train_per_sec_per_gpu': '33.87', 'tokens/total': 6921776, 'tokens/trainable': 104796, 'epoch': '0.8945'} |
|
45%|████▍ | 229/512 [25:27<31:44, 6.73s/it]
45%|████▍ | 230/512 [25:33<31:31, 6.71s/it]
{'loss': '0.01232', 'grad_norm': '0.4787', 'learning_rate': '6.634e-05', 'ppl': '1.012', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.86', 'tokens/train_per_sec_per_gpu': '37.27', 'tokens/total': 6952048, 'tokens/trainable': 105264, 'epoch': '0.8984'} |
|
45%|████▍ | 230/512 [25:33<31:31, 6.71s/it]
45%|████▌ | 231/512 [25:40<31:15, 6.68s/it]
{'loss': '0.008309', 'grad_norm': '0.4259', 'learning_rate': '6.606e-05', 'ppl': '1.008', 'memory/max_active (GiB)': '33.79', 'memory/max_allocated (GiB)': '33.79', 'memory/device_reserved (GiB)': '35.86', 'tokens/train_per_sec_per_gpu': '34.25', 'tokens/total': 6982384, 'tokens/trainable': 105727, 'epoch': '0.9023'} |
|
45%|████▌ | 231/512 [25:40<31:15, 6.68s/it]
45%|████▌ | 232/512 [25:47<31:03, 6.66s/it]
{'loss': '0.001102', 'grad_norm': '0.07725', 'learning_rate': '6.578e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '33.45', 'tokens/total': 7012784, 'tokens/trainable': 106203, 'epoch': '0.9062'} |
|
45%|████▌ | 232/512 [25:47<31:03, 6.66s/it]
46%|████▌ | 233/512 [25:53<30:57, 6.66s/it]
{'loss': '0.001225', 'grad_norm': '0.07385', 'learning_rate': '6.55e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '38.46', 'tokens/total': 7043184, 'tokens/trainable': 106708, 'epoch': '0.9102'} |
|
46%|████▌ | 233/512 [25:53<30:57, 6.66s/it]
46%|████▌ | 234/512 [26:00<30:44, 6.63s/it]
{'loss': '0.006349', 'grad_norm': '0.3441', 'learning_rate': '6.522e-05', 'ppl': '1.006', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '34.19', 'tokens/total': 7073168, 'tokens/trainable': 107146, 'epoch': '0.9141'} |
|
46%|████▌ | 234/512 [26:00<30:44, 6.63s/it]
46%|████▌ | 235/512 [26:06<30:36, 6.63s/it]
{'loss': '0.0003933', 'grad_norm': '0.01911', 'learning_rate': '6.493e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '33.1', 'tokens/total': 7103616, 'tokens/trainable': 107586, 'epoch': '0.918'} |
|
46%|████▌ | 235/512 [26:06<30:36, 6.63s/it]
46%|████▌ | 236/512 [26:13<30:31, 6.64s/it]
{'loss': '0.001894', 'grad_norm': '0.1013', 'learning_rate': '6.465e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '34.29', 'tokens/total': 7133840, 'tokens/trainable': 108051, 'epoch': '0.9219'} |
|
46%|████▌ | 236/512 [26:13<30:31, 6.64s/it]
46%|████▋ | 237/512 [26:20<30:20, 6.62s/it]
{'loss': '0.0005699', 'grad_norm': '0.07892', 'learning_rate': '6.437e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.77', 'memory/max_allocated (GiB)': '33.77', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '33.72', 'tokens/total': 7164016, 'tokens/trainable': 108479, 'epoch': '0.9258'} |
|
46%|████▋ | 237/512 [26:20<30:20, 6.62s/it]
46%|████▋ | 238/512 [26:26<30:09, 6.61s/it]
{'loss': '0.01608', 'grad_norm': '0.4524', 'learning_rate': '6.408e-05', 'ppl': '1.016', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '29.52', 'tokens/total': 7194400, 'tokens/trainable': 108907, 'epoch': '0.9297'} |
|
46%|████▋ | 238/512 [26:26<30:09, 6.61s/it]
47%|████▋ | 239/512 [26:33<30:09, 6.63s/it]
{'loss': '0.009672', 'grad_norm': '0.2393', 'learning_rate': '6.38e-05', 'ppl': '1.01', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '32.81', 'tokens/total': 7224992, 'tokens/trainable': 109348, 'epoch': '0.9336'} |
|
47%|████▋ | 239/512 [26:33<30:09, 6.63s/it]
47%|████▋ | 240/512 [26:40<30:03, 6.63s/it]
{'loss': '0.001241', 'grad_norm': '0.04966', 'learning_rate': '6.351e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '32.71', 'tokens/total': 7255424, 'tokens/trainable': 109812, 'epoch': '0.9375'} |
|
47%|████▋ | 240/512 [26:40<30:03, 6.63s/it]
47%|████▋ | 241/512 [26:46<29:50, 6.61s/it]
{'loss': '0.00566', 'grad_norm': '0.2983', 'learning_rate': '6.323e-05', 'ppl': '1.006', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '38.88', 'tokens/total': 7285696, 'tokens/trainable': 110270, 'epoch': '0.9414'} |
|
47%|████▋ | 241/512 [26:46<29:50, 6.61s/it]
47%|████▋ | 242/512 [26:53<29:45, 6.61s/it]
{'loss': '0.004708', 'grad_norm': '0.2699', 'learning_rate': '6.294e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '28.25', 'tokens/total': 7316000, 'tokens/trainable': 110682, 'epoch': '0.9453'} |
|
47%|████▋ | 242/512 [26:53<29:45, 6.61s/it]
47%|████▋ | 243/512 [26:59<29:41, 6.62s/it]
{'loss': '0.003754', 'grad_norm': '0.4308', 'learning_rate': '6.266e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '39.54', 'tokens/total': 7346304, 'tokens/trainable': 111176, 'epoch': '0.9492'} |
|
47%|████▋ | 243/512 [26:59<29:41, 6.62s/it]
48%|████▊ | 244/512 [27:06<29:40, 6.64s/it]
{'loss': '0.008292', 'grad_norm': '0.277', 'learning_rate': '6.237e-05', 'ppl': '1.008', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '33.34', 'tokens/total': 7376944, 'tokens/trainable': 111633, 'epoch': '0.9531'} |
|
48%|████▊ | 244/512 [27:06<29:40, 6.64s/it]
48%|████▊ | 245/512 [27:13<29:29, 6.63s/it]
{'loss': '0.001885', 'grad_norm': '0.0792', 'learning_rate': '6.208e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.72', 'memory/max_allocated (GiB)': '33.72', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '34.65', 'tokens/total': 7407184, 'tokens/trainable': 112115, 'epoch': '0.957'} |
|
48%|████▊ | 245/512 [27:13<29:29, 6.63s/it]
48%|████▊ | 246/512 [27:19<29:21, 6.62s/it]
{'loss': '0.008208', 'grad_norm': '0.3069', 'learning_rate': '6.18e-05', 'ppl': '1.008', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '33.59', 'tokens/total': 7437760, 'tokens/trainable': 112589, 'epoch': '0.9609'} |
|
48%|████▊ | 246/512 [27:19<29:21, 6.62s/it]
48%|████▊ | 247/512 [27:25<28:37, 6.48s/it]
{'loss': '0.004535', 'grad_norm': '0.2259', 'learning_rate': '6.151e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '38.2', 'tokens/total': 7466080, 'tokens/trainable': 113043, 'epoch': '0.9648'} |
|
48%|████▊ | 247/512 [27:25<28:37, 6.48s/it]
48%|████▊ | 248/512 [27:32<28:46, 6.54s/it]
{'loss': '0.002737', 'grad_norm': '0.1515', 'learning_rate': '6.122e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '35.66', 'tokens/total': 7496672, 'tokens/trainable': 113484, 'epoch': '0.9688'} |
|
48%|████▊ | 248/512 [27:32<28:46, 6.54s/it]
49%|████▊ | 249/512 [27:39<28:44, 6.56s/it]
{'loss': '0.002225', 'grad_norm': '0.09306', 'learning_rate': '6.093e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '33.02', 'tokens/total': 7526816, 'tokens/trainable': 113928, 'epoch': '0.9727'} |
|
49%|████▊ | 249/512 [27:39<28:44, 6.56s/it]
49%|████▉ | 250/512 [27:45<28:07, 6.44s/it]
{'loss': '0.0009518', 'grad_norm': '0.06742', 'learning_rate': '6.065e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '34.24', 'tokens/total': 7555120, 'tokens/trainable': 114360, 'epoch': '0.9766'} |
|
49%|████▉ | 250/512 [27:45<28:07, 6.44s/it]
49%|████▉ | 251/512 [27:51<28:15, 6.50s/it]
{'loss': '0.004424', 'grad_norm': '0.09548', 'learning_rate': '6.036e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '36.82', 'tokens/total': 7585632, 'tokens/trainable': 114850, 'epoch': '0.9805'} |
|
49%|████▉ | 251/512 [27:51<28:15, 6.50s/it]
49%|████▉ | 252/512 [27:58<28:20, 6.54s/it]
{'loss': '0.009106', 'grad_norm': '0.3539', 'learning_rate': '6.007e-05', 'ppl': '1.009', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '35.56', 'tokens/total': 7615888, 'tokens/trainable': 115298, 'epoch': '0.9844'} |
|
49%|████▉ | 252/512 [27:58<28:20, 6.54s/it]
49%|████▉ | 253/512 [28:05<28:25, 6.58s/it]
{'loss': '0.0007137', 'grad_norm': '0.04793', 'learning_rate': '5.978e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '32.61', 'tokens/total': 7646464, 'tokens/trainable': 115719, 'epoch': '0.9883'} |
|
49%|████▉ | 253/512 [28:05<28:25, 6.58s/it]
50%|████▉ | 254/512 [28:11<28:24, 6.61s/it]
{'loss': '0.005779', 'grad_norm': '0.2061', 'learning_rate': '5.949e-05', 'ppl': '1.006', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '29.56', 'tokens/total': 7676896, 'tokens/trainable': 116151, 'epoch': '0.9922'} |
|
50%|████▉ | 254/512 [28:11<28:24, 6.61s/it]
50%|████▉ | 255/512 [28:18<28:18, 6.61s/it]
{'loss': '0.0005411', 'grad_norm': '0.02974', 'learning_rate': '5.92e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '35.94', 'tokens/train_per_sec_per_gpu': '37.19', 'tokens/total': 7707328, 'tokens/trainable': 116602, 'epoch': '0.9961'} |
|
50%|████▉ | 255/512 [28:18<28:18, 6.61s/it]
50%|█████ | 256/512 [28:25<28:26, 6.67s/it]
{'loss': '0.0003776', 'grad_norm': '0.02681', 'learning_rate': '5.891e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.98', 'memory/max_allocated (GiB)': '33.98', 'memory/device_reserved (GiB)': '36.21', 'tokens/train_per_sec_per_gpu': '32.45', 'tokens/total': 7737920, 'tokens/trainable': 117050, 'epoch': '1'} |
|
50%|█████ | 256/512 [28:25<28:26, 6.67s/it][2026-08-18 15:00:08,523] [INFO] [axolotl.core.trainers.base._save:828] [PID:12035] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-256 |
|
50%|█████ | 257/512 [28:35<32:12, 7.58s/it]
{'loss': '0.001542', 'grad_norm': '0.1312', 'learning_rate': '5.862e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '36.21', 'tokens/train_per_sec_per_gpu': '32.54', 'tokens/total': 7768256, 'tokens/trainable': 117506, 'epoch': '1.004'} |
|
50%|█████ | 257/512 [28:35<32:12, 7.58s/it]
50%|█████ | 258/512 [28:41<30:51, 7.29s/it]
{'loss': '0.0001414', 'grad_norm': '0.00715', 'learning_rate': '5.834e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '30.82', 'tokens/total': 7798544, 'tokens/trainable': 117936, 'epoch': '1.008'} |
|
50%|█████ | 258/512 [28:41<30:51, 7.29s/it]
51%|█████ | 259/512 [28:48<29:54, 7.09s/it]
{'loss': '0.002642', 'grad_norm': '0.2526', 'learning_rate': '5.805e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '31.96', 'tokens/total': 7828880, 'tokens/trainable': 118377, 'epoch': '1.012'} |
|
51%|█████ | 259/512 [28:48<29:54, 7.09s/it]
51%|█████ | 260/512 [28:54<29:11, 6.95s/it]
{'loss': '4.687e-05', 'grad_norm': '0.002305', 'learning_rate': '5.776e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '31.93', 'tokens/total': 7859312, 'tokens/trainable': 118820, 'epoch': '1.016'} |
|
51%|█████ | 260/512 [28:54<29:11, 6.95s/it]
51%|█████ | 261/512 [29:01<28:36, 6.84s/it]
{'loss': '0.003436', 'grad_norm': '0.1961', 'learning_rate': '5.747e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '34.37', 'tokens/total': 7889776, 'tokens/trainable': 119267, 'epoch': '1.02'} |
|
51%|█████ | 261/512 [29:01<28:36, 6.84s/it]
51%|█████ | 262/512 [29:08<28:17, 6.79s/it]
{'loss': '0.0006204', 'grad_norm': '0.06521', 'learning_rate': '5.718e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '35.19', 'tokens/total': 7920256, 'tokens/trainable': 119776, 'epoch': '1.023'} |
|
51%|█████ | 262/512 [29:08<28:17, 6.79s/it]
51%|█████▏ | 263/512 [29:14<27:59, 6.75s/it]
{'loss': '0.003635', 'grad_norm': '0.238', 'learning_rate': '5.689e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '34.33', 'tokens/total': 7950608, 'tokens/trainable': 120265, 'epoch': '1.027'} |
|
51%|█████▏ | 263/512 [29:14<27:59, 6.75s/it]
52%|█████▏ | 264/512 [29:21<27:46, 6.72s/it]
{'loss': '0.002851', 'grad_norm': '0.2221', 'learning_rate': '5.66e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '37.54', 'tokens/total': 7981168, 'tokens/trainable': 120742, 'epoch': '1.031'} |
|
52%|█████▏ | 264/512 [29:21<27:46, 6.72s/it]
52%|█████▏ | 265/512 [29:28<27:30, 6.68s/it]
{'loss': '0.0001093', 'grad_norm': '0.00784', 'learning_rate': '5.631e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '40.03', 'tokens/total': 8011504, 'tokens/trainable': 121209, 'epoch': '1.035'} |
|
52%|█████▏ | 265/512 [29:28<27:30, 6.68s/it]
52%|█████▏ | 266/512 [29:34<27:25, 6.69s/it]
{'loss': '0.0008161', 'grad_norm': '0.06024', 'learning_rate': '5.602e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '32.67', 'tokens/total': 8042048, 'tokens/trainable': 121683, 'epoch': '1.039'} |
|
52%|█████▏ | 266/512 [29:34<27:25, 6.69s/it]
52%|█████▏ | 267/512 [29:41<27:17, 6.68s/it]
{'loss': '0.0001045', 'grad_norm': '0.01427', 'learning_rate': '5.573e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '40.53', 'tokens/total': 8072336, 'tokens/trainable': 122187, 'epoch': '1.043'} |
|
52%|█████▏ | 267/512 [29:41<27:17, 6.68s/it]
52%|█████▏ | 268/512 [29:48<27:04, 6.66s/it]
{'loss': '0.0003805', 'grad_norm': '0.03649', 'learning_rate': '5.544e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '35.89', 'tokens/total': 8102592, 'tokens/trainable': 122649, 'epoch': '1.047'} |
|
52%|█████▏ | 268/512 [29:48<27:04, 6.66s/it]
53%|█████▎ | 269/512 [29:54<26:53, 6.64s/it]
{'loss': '0.0005166', 'grad_norm': '0.04869', 'learning_rate': '5.515e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.71', 'memory/max_allocated (GiB)': '33.71', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '33.52', 'tokens/total': 8132752, 'tokens/trainable': 123109, 'epoch': '1.051'} |
|
53%|█████▎ | 269/512 [29:54<26:53, 6.64s/it]
53%|█████▎ | 270/512 [30:01<26:47, 6.64s/it]
{'loss': '0.0009587', 'grad_norm': '0.08865', 'learning_rate': '5.485e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '36.69', 'tokens/total': 8163232, 'tokens/trainable': 123599, 'epoch': '1.055'} |
|
53%|█████▎ | 270/512 [30:01<26:47, 6.64s/it]
53%|█████▎ | 271/512 [30:07<26:38, 6.63s/it]
{'loss': '0.0005165', 'grad_norm': '0.07191', 'learning_rate': '5.456e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '38.15', 'tokens/total': 8193344, 'tokens/trainable': 124081, 'epoch': '1.059'} |
|
53%|█████▎ | 271/512 [30:07<26:38, 6.63s/it]
53%|█████▎ | 272/512 [30:14<26:33, 6.64s/it]
{'loss': '0.0004327', 'grad_norm': '0.06801', 'learning_rate': '5.427e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '35.19', 'tokens/total': 8223632, 'tokens/trainable': 124551, 'epoch': '1.062'} |
|
53%|█████▎ | 272/512 [30:14<26:33, 6.64s/it]
53%|█████▎ | 273/512 [30:21<26:22, 6.62s/it]
{'loss': '5.501e-05', 'grad_norm': '0.008596', 'learning_rate': '5.398e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.71', 'memory/max_allocated (GiB)': '33.71', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '34.47', 'tokens/total': 8253680, 'tokens/trainable': 124981, 'epoch': '1.066'} |
|
53%|█████▎ | 273/512 [30:21<26:22, 6.62s/it]
54%|█████▎ | 274/512 [30:27<26:14, 6.62s/it]
{'loss': '0.0006859', 'grad_norm': '0.1259', 'learning_rate': '5.369e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.9', 'tokens/train_per_sec_per_gpu': '35', 'tokens/total': 8284256, 'tokens/trainable': 125442, 'epoch': '1.07'} |
|
54%|█████▎ | 274/512 [30:27<26:14, 6.62s/it]
54%|█████▎ | 275/512 [30:34<26:11, 6.63s/it]
{'loss': '2.156e-05', 'grad_norm': '0.002709', 'learning_rate': '5.34e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.98', 'memory/max_allocated (GiB)': '33.98', 'memory/device_reserved (GiB)': '35.9', 'tokens/train_per_sec_per_gpu': '31.81', 'tokens/total': 8314896, 'tokens/trainable': 125893, 'epoch': '1.074'} |
|
54%|█████▎ | 275/512 [30:34<26:11, 6.63s/it]
54%|█████▍ | 276/512 [30:41<26:07, 6.64s/it]
{'loss': '0.003218', 'grad_norm': '0.65', 'learning_rate': '5.311e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.9', 'tokens/train_per_sec_per_gpu': '33.15', 'tokens/total': 8345264, 'tokens/trainable': 126367, 'epoch': '1.078'} |
|
54%|█████▍ | 276/512 [30:41<26:07, 6.64s/it]
54%|█████▍ | 277/512 [30:47<26:04, 6.66s/it]
{'loss': '4.369e-05', 'grad_norm': '0.003593', 'learning_rate': '5.282e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.9', 'tokens/train_per_sec_per_gpu': '34.39', 'tokens/total': 8375728, 'tokens/trainable': 126832, 'epoch': '1.082'} |
|
54%|█████▍ | 277/512 [30:47<26:04, 6.66s/it]
54%|█████▍ | 278/512 [30:54<25:54, 6.64s/it]
{'loss': '0.0003241', 'grad_norm': '0.06011', 'learning_rate': '5.253e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '35.9', 'tokens/train_per_sec_per_gpu': '38.74', 'tokens/total': 8405952, 'tokens/trainable': 127335, 'epoch': '1.086'} |
|
54%|█████▍ | 278/512 [30:54<25:54, 6.64s/it]
54%|█████▍ | 279/512 [31:01<25:45, 6.63s/it]
{'loss': '4.077e-05', 'grad_norm': '0.004136', 'learning_rate': '5.224e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.77', 'memory/max_allocated (GiB)': '33.77', 'memory/device_reserved (GiB)': '35.9', 'tokens/train_per_sec_per_gpu': '33.47', 'tokens/total': 8436096, 'tokens/trainable': 127793, 'epoch': '1.09'} |
|
54%|█████▍ | 279/512 [31:01<25:45, 6.63s/it]
55%|█████▍ | 280/512 [31:07<25:38, 6.63s/it]
{'loss': '2.093e-05', 'grad_norm': '0.001947', 'learning_rate': '5.195e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.9', 'tokens/train_per_sec_per_gpu': '33.21', 'tokens/total': 8466464, 'tokens/trainable': 128242, 'epoch': '1.094'} |
|
55%|█████▍ | 280/512 [31:07<25:38, 6.63s/it]
55%|█████▍ | 281/512 [31:14<25:33, 6.64s/it]
{'loss': '2.148e-05', 'grad_norm': '0.001756', 'learning_rate': '5.166e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.9', 'tokens/train_per_sec_per_gpu': '36.63', 'tokens/total': 8496736, 'tokens/trainable': 128724, 'epoch': '1.098'} |
|
55%|█████▍ | 281/512 [31:14<25:33, 6.64s/it]
55%|█████▌ | 282/512 [31:20<25:27, 6.64s/it]
{'loss': '0.0001425', 'grad_norm': '0.01728', 'learning_rate': '5.138e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.9', 'tokens/train_per_sec_per_gpu': '35.5', 'tokens/total': 8526928, 'tokens/trainable': 129198, 'epoch': '1.102'} |
|
55%|█████▌ | 282/512 [31:20<25:27, 6.64s/it]
55%|█████▌ | 283/512 [31:27<25:21, 6.65s/it]
{'loss': '6.786e-05', 'grad_norm': '0.01479', 'learning_rate': '5.109e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.91', 'tokens/train_per_sec_per_gpu': '33.38', 'tokens/total': 8557296, 'tokens/trainable': 129648, 'epoch': '1.105'} |
|
55%|█████▌ | 283/512 [31:27<25:21, 6.65s/it]
55%|█████▌ | 284/512 [31:34<25:20, 6.67s/it]
{'loss': '0.004038', 'grad_norm': '0.4628', 'learning_rate': '5.08e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '35.91', 'tokens/train_per_sec_per_gpu': '32.72', 'tokens/total': 8587696, 'tokens/trainable': 130095, 'epoch': '1.109'} |
|
55%|█████▌ | 284/512 [31:34<25:20, 6.67s/it]
56%|█████▌ | 285/512 [31:40<25:12, 6.66s/it]
{'loss': '0.002691', 'grad_norm': '0.3512', 'learning_rate': '5.051e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '35.91', 'tokens/train_per_sec_per_gpu': '31.74', 'tokens/total': 8618080, 'tokens/trainable': 130545, 'epoch': '1.113'} |
|
56%|█████▌ | 285/512 [31:40<25:12, 6.66s/it]
56%|█████▌ | 286/512 [31:47<25:04, 6.66s/it]
{'loss': '8.888e-05', 'grad_norm': '0.009707', 'learning_rate': '5.022e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.91', 'tokens/train_per_sec_per_gpu': '34.64', 'tokens/total': 8648288, 'tokens/trainable': 131002, 'epoch': '1.117'} |
|
56%|█████▌ | 286/512 [31:47<25:04, 6.66s/it]
56%|█████▌ | 287/512 [31:54<24:57, 6.65s/it]
{'loss': '0.000788', 'grad_norm': '0.07988', 'learning_rate': '4.993e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.79', 'memory/max_allocated (GiB)': '33.79', 'memory/device_reserved (GiB)': '35.91', 'tokens/train_per_sec_per_gpu': '34.31', 'tokens/total': 8678464, 'tokens/trainable': 131458, 'epoch': '1.121'} |
|
56%|█████▌ | 287/512 [31:54<24:57, 6.65s/it]
56%|█████▋ | 288/512 [32:00<24:47, 6.64s/it]
{'loss': '1.423e-05', 'grad_norm': '0.001365', 'learning_rate': '4.964e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '35.91', 'tokens/train_per_sec_per_gpu': '36.32', 'tokens/total': 8708816, 'tokens/trainable': 131932, 'epoch': '1.125'} |
|
56%|█████▋ | 288/512 [32:00<24:47, 6.64s/it][2026-08-18 15:03:44,035] [INFO] [axolotl.core.trainers.base._save:828] [PID:12035] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-288 |
|
56%|█████▋ | 289/512 [32:09<26:54, 7.24s/it]
{'loss': '5.348e-06', 'grad_norm': '0.0002254', 'learning_rate': '4.935e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '35.91', 'tokens/train_per_sec_per_gpu': '33.57', 'tokens/total': 8738992, 'tokens/trainable': 132406, 'epoch': '1.129'} |
|
56%|█████▋ | 289/512 [32:09<26:54, 7.24s/it]
57%|█████▋ | 290/512 [32:16<26:06, 7.06s/it]
{'loss': '6.508e-06', 'grad_norm': '0.0003425', 'learning_rate': '4.907e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '35.57', 'tokens/train_per_sec_per_gpu': '36.83', 'tokens/total': 8769424, 'tokens/trainable': 132855, 'epoch': '1.133'} |
|
57%|█████▋ | 290/512 [32:16<26:06, 7.06s/it]
57%|█████▋ | 291/512 [32:22<25:31, 6.93s/it]
{'loss': '2.1e-05', 'grad_norm': '0.001163', 'learning_rate': '4.878e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '35.57', 'tokens/train_per_sec_per_gpu': '36.18', 'tokens/total': 8799680, 'tokens/trainable': 133356, 'epoch': '1.137'} |
|
57%|█████▋ | 291/512 [32:22<25:31, 6.93s/it]
57%|█████▋ | 292/512 [32:29<25:04, 6.84s/it]
{'loss': '5.513e-06', 'grad_norm': '0.0002512', 'learning_rate': '4.849e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '35.59', 'tokens/train_per_sec_per_gpu': '34.54', 'tokens/total': 8830000, 'tokens/trainable': 133788, 'epoch': '1.141'} |
|
57%|█████▋ | 292/512 [32:29<25:04, 6.84s/it]
57%|█████▋ | 293/512 [32:36<24:47, 6.79s/it]
{'loss': '5.693e-05', 'grad_norm': '0.005343', 'learning_rate': '4.82e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.59', 'tokens/train_per_sec_per_gpu': '36.67', 'tokens/total': 8860320, 'tokens/trainable': 134260, 'epoch': '1.145'} |
|
57%|█████▋ | 293/512 [32:36<24:47, 6.79s/it]
57%|█████▋ | 294/512 [32:42<24:33, 6.76s/it]
{'loss': '5.992e-05', 'grad_norm': '0.01513', 'learning_rate': '4.792e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '35.59', 'tokens/train_per_sec_per_gpu': '34.01', 'tokens/total': 8891072, 'tokens/trainable': 134684, 'epoch': '1.148'} |
|
57%|█████▋ | 294/512 [32:42<24:33, 6.76s/it]
58%|█████▊ | 295/512 [32:49<24:18, 6.72s/it]
{'loss': '0.001324', 'grad_norm': '0.2628', 'learning_rate': '4.763e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.59', 'tokens/train_per_sec_per_gpu': '36.05', 'tokens/total': 8921504, 'tokens/trainable': 135166, 'epoch': '1.152'} |
|
58%|█████▊ | 295/512 [32:49<24:18, 6.72s/it]
58%|█████▊ | 296/512 [32:55<24:02, 6.68s/it]
{'loss': '4.622e-06', 'grad_norm': '0.0001754', 'learning_rate': '4.734e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '35.67', 'tokens/train_per_sec_per_gpu': '33.62', 'tokens/total': 8951808, 'tokens/trainable': 135600, 'epoch': '1.156'} |
|
58%|█████▊ | 296/512 [32:55<24:02, 6.68s/it]
58%|█████▊ | 297/512 [33:02<23:51, 6.66s/it]
{'loss': '0.007754', 'grad_norm': '0.344', 'learning_rate': '4.706e-05', 'ppl': '1.008', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '35.67', 'tokens/train_per_sec_per_gpu': '32.99', 'tokens/total': 8982080, 'tokens/trainable': 136039, 'epoch': '1.16'} |
|
58%|█████▊ | 297/512 [33:02<23:51, 6.66s/it]
58%|█████▊ | 298/512 [33:09<23:43, 6.65s/it]
{'loss': '5.842e-05', 'grad_norm': '0.01015', 'learning_rate': '4.677e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.67', 'tokens/train_per_sec_per_gpu': '33.31', 'tokens/total': 9012560, 'tokens/trainable': 136496, 'epoch': '1.164'} |
|
58%|█████▊ | 298/512 [33:09<23:43, 6.65s/it]
58%|█████▊ | 299/512 [33:15<23:37, 6.65s/it]
{'loss': '3.438e-05', 'grad_norm': '0.003886', 'learning_rate': '4.649e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.67', 'tokens/train_per_sec_per_gpu': '35.75', 'tokens/total': 9042944, 'tokens/trainable': 136954, 'epoch': '1.168'} |
|
58%|█████▊ | 299/512 [33:15<23:37, 6.65s/it]
59%|█████▊ | 300/512 [33:22<23:26, 6.63s/it]
{'loss': '2.051e-05', 'grad_norm': '0.001525', 'learning_rate': '4.62e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.77', 'memory/max_allocated (GiB)': '33.77', 'memory/device_reserved (GiB)': '35.67', 'tokens/train_per_sec_per_gpu': '34.64', 'tokens/total': 9073248, 'tokens/trainable': 137404, 'epoch': '1.172'} |
|
59%|█████▊ | 300/512 [33:22<23:26, 6.63s/it]
59%|█████▉ | 301/512 [33:29<23:18, 6.63s/it]
{'loss': '0.006097', 'grad_norm': '0.4778', 'learning_rate': '4.592e-05', 'ppl': '1.006', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '35.67', 'tokens/train_per_sec_per_gpu': '36.12', 'tokens/total': 9103728, 'tokens/trainable': 137899, 'epoch': '1.176'} |
|
59%|█████▉ | 301/512 [33:29<23:18, 6.63s/it]
59%|█████▉ | 302/512 [33:35<23:16, 6.65s/it]
{'loss': '0.0004718', 'grad_norm': '0.09574', 'learning_rate': '4.563e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.67', 'tokens/train_per_sec_per_gpu': '34.52', 'tokens/total': 9134000, 'tokens/trainable': 138375, 'epoch': '1.18'} |
|
59%|█████▉ | 302/512 [33:35<23:16, 6.65s/it]
59%|█████▉ | 303/512 [33:42<23:11, 6.66s/it]
{'loss': '4.185e-05', 'grad_norm': '0.003621', 'learning_rate': '4.535e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.67', 'tokens/train_per_sec_per_gpu': '36.82', 'tokens/total': 9164400, 'tokens/trainable': 138834, 'epoch': '1.184'} |
|
59%|█████▉ | 303/512 [33:42<23:11, 6.66s/it]
59%|█████▉ | 304/512 [33:49<23:00, 6.64s/it]
{'loss': '3.689e-05', 'grad_norm': '0.004096', 'learning_rate': '4.507e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '35.67', 'tokens/train_per_sec_per_gpu': '30.99', 'tokens/total': 9194672, 'tokens/trainable': 139259, 'epoch': '1.188'} |
|
59%|█████▉ | 304/512 [33:49<23:00, 6.64s/it]
60%|█████▉ | 305/512 [33:55<22:54, 6.64s/it]
{'loss': '4.464e-05', 'grad_norm': '0.004101', 'learning_rate': '4.478e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.67', 'tokens/train_per_sec_per_gpu': '33.42', 'tokens/total': 9224848, 'tokens/trainable': 139717, 'epoch': '1.191'} |
|
60%|█████▉ | 305/512 [33:55<22:54, 6.64s/it]
60%|█████▉ | 306/512 [34:02<22:47, 6.64s/it]
{'loss': '0.002607', 'grad_norm': '0.2227', 'learning_rate': '4.45e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '35.9', 'tokens/train_per_sec_per_gpu': '37.37', 'tokens/total': 9255440, 'tokens/trainable': 140207, 'epoch': '1.195'} |
|
60%|█████▉ | 306/512 [34:02<22:47, 6.64s/it]
60%|█████▉ | 307/512 [34:08<22:03, 6.46s/it]
{'loss': '0.0004709', 'grad_norm': '0.05723', 'learning_rate': '4.422e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.19', 'memory/max_allocated (GiB)': '33.19', 'memory/device_reserved (GiB)': '35.9', 'tokens/train_per_sec_per_gpu': '34.35', 'tokens/total': 9283344, 'tokens/trainable': 140645, 'epoch': '1.199'} |
|
60%|█████▉ | 307/512 [34:08<22:03, 6.46s/it]
60%|██████ | 308/512 [34:15<22:11, 6.53s/it]
{'loss': '0.004533', 'grad_norm': '0.694', 'learning_rate': '4.394e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '35.9', 'tokens/train_per_sec_per_gpu': '30.34', 'tokens/total': 9313856, 'tokens/trainable': 141096, 'epoch': '1.203'} |
|
60%|██████ | 308/512 [34:15<22:11, 6.53s/it]
60%|██████ | 309/512 [34:21<22:11, 6.56s/it]
{'loss': '4.202e-05', 'grad_norm': '0.003843', 'learning_rate': '4.366e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '29.26', 'tokens/total': 9344064, 'tokens/trainable': 141518, 'epoch': '1.207'} |
|
60%|██████ | 309/512 [34:21<22:11, 6.56s/it]
61%|██████ | 310/512 [34:28<22:06, 6.57s/it]
{'loss': '4.229e-05', 'grad_norm': '0.002202', 'learning_rate': '4.338e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '35.84', 'tokens/total': 9374368, 'tokens/trainable': 141952, 'epoch': '1.211'} |
|
61%|██████ | 310/512 [34:28<22:06, 6.57s/it]
61%|██████ | 311/512 [34:34<22:02, 6.58s/it]
{'loss': '1.288e-05', 'grad_norm': '0.0005679', 'learning_rate': '4.31e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '34.36', 'tokens/total': 9404624, 'tokens/trainable': 142416, 'epoch': '1.215'} |
|
61%|██████ | 311/512 [34:34<22:02, 6.58s/it]
61%|██████ | 312/512 [34:41<21:54, 6.57s/it]
{'loss': '1.209e-05', 'grad_norm': '0.0005692', 'learning_rate': '4.282e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '38.63', 'tokens/total': 9434784, 'tokens/trainable': 142882, 'epoch': '1.219'} |
|
61%|██████ | 312/512 [34:41<21:54, 6.57s/it]
61%|██████ | 313/512 [34:48<21:50, 6.59s/it]
{'loss': '9.982e-05', 'grad_norm': '0.008793', 'learning_rate': '4.254e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '36.84', 'tokens/total': 9465168, 'tokens/trainable': 143356, 'epoch': '1.223'} |
|
61%|██████ | 313/512 [34:48<21:50, 6.59s/it]
61%|██████▏ | 314/512 [34:54<21:45, 6.59s/it]
{'loss': '0.0001094', 'grad_norm': '0.008733', 'learning_rate': '4.226e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.79', 'memory/max_allocated (GiB)': '33.79', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '33.31', 'tokens/total': 9495488, 'tokens/trainable': 143812, 'epoch': '1.227'} |
|
61%|██████▏ | 314/512 [34:54<21:45, 6.59s/it]
62%|██████▏ | 315/512 [35:01<21:46, 6.63s/it]
{'loss': '0.0001752', 'grad_norm': '0.01676', 'learning_rate': '4.198e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.99', 'memory/max_allocated (GiB)': '33.99', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '34.95', 'tokens/total': 9525968, 'tokens/trainable': 144294, 'epoch': '1.23'} |
|
62%|██████▏ | 315/512 [35:01<21:46, 6.63s/it]
62%|██████▏ | 316/512 [35:07<21:38, 6.63s/it]
{'loss': '0.0001436', 'grad_norm': '0.01962', 'learning_rate': '4.17e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '35.87', 'tokens/total': 9556368, 'tokens/trainable': 144751, 'epoch': '1.234'} |
|
62%|██████▏ | 316/512 [35:07<21:38, 6.63s/it]
62%|██████▏ | 317/512 [35:14<21:35, 6.65s/it]
{'loss': '3.047e-05', 'grad_norm': '0.00189', 'learning_rate': '4.143e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '37.67', 'tokens/total': 9586800, 'tokens/trainable': 145251, 'epoch': '1.238'} |
|
62%|██████▏ | 317/512 [35:14<21:35, 6.65s/it]
62%|██████▏ | 318/512 [35:21<21:29, 6.65s/it]
{'loss': '0.0003194', 'grad_norm': '0.0303', 'learning_rate': '4.115e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '37.13', 'tokens/total': 9617152, 'tokens/trainable': 145711, 'epoch': '1.242'} |
|
62%|██████▏ | 318/512 [35:21<21:29, 6.65s/it]
62%|██████▏ | 319/512 [35:27<21:23, 6.65s/it]
{'loss': '0.000548', 'grad_norm': '0.04471', 'learning_rate': '4.087e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '37.59', 'tokens/total': 9647424, 'tokens/trainable': 146205, 'epoch': '1.246'} |
|
62%|██████▏ | 319/512 [35:27<21:23, 6.65s/it]
62%|██████▎ | 320/512 [35:34<21:16, 6.65s/it]
{'loss': '0.003161', 'grad_norm': '1.604', 'learning_rate': '4.06e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '34.32', 'tokens/total': 9677776, 'tokens/trainable': 146678, 'epoch': '1.25'} |
|
62%|██████▎ | 320/512 [35:34<21:16, 6.65s/it][2026-08-18 15:07:17,797] [INFO] [axolotl.core.trainers.base._save:828] [PID:12035] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-320 |
|
63%|██████▎ | 321/512 [35:42<22:44, 7.14s/it]
{'loss': '3.962e-05', 'grad_norm': '0.003327', 'learning_rate': '4.032e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '35.71', 'tokens/total': 9707936, 'tokens/trainable': 147124, 'epoch': '1.254'} |
|
63%|██████▎ | 321/512 [35:42<22:44, 7.14s/it]
63%|██████▎ | 322/512 [35:49<22:06, 6.98s/it]
{'loss': '7.033e-05', 'grad_norm': '0.005384', 'learning_rate': '4.005e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '35.58', 'tokens/total': 9738448, 'tokens/trainable': 147574, 'epoch': '1.258'} |
|
63%|██████▎ | 322/512 [35:49<22:06, 6.98s/it]
63%|██████▎ | 323/512 [35:56<21:39, 6.87s/it]
{'loss': '5.104e-05', 'grad_norm': '0.002838', 'learning_rate': '3.978e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '35.65', 'tokens/total': 9768864, 'tokens/trainable': 148030, 'epoch': '1.262'} |
|
63%|██████▎ | 323/512 [35:56<21:39, 6.87s/it]
63%|██████▎ | 324/512 [36:02<21:21, 6.82s/it]
{'loss': '0.0001162', 'grad_norm': '0.02523', 'learning_rate': '3.95e-05', 'ppl': '1', 'memory/max_active (GiB)': '34.01', 'memory/max_allocated (GiB)': '34.01', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '37.22', 'tokens/total': 9799184, 'tokens/trainable': 148478, 'epoch': '1.266'} |
|
63%|██████▎ | 324/512 [36:02<21:21, 6.82s/it]
63%|██████▎ | 325/512 [36:09<21:01, 6.75s/it]
{'loss': '0.0003832', 'grad_norm': '0.0495', 'learning_rate': '3.923e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '36.95', 'tokens/total': 9829392, 'tokens/trainable': 148941, 'epoch': '1.27'} |
|
63%|██████▎ | 325/512 [36:09<21:01, 6.75s/it]
64%|██████▎ | 326/512 [36:16<20:51, 6.73s/it]
{'loss': '2.421e-05', 'grad_norm': '0.001763', 'learning_rate': '3.896e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '38.19', 'tokens/total': 9859696, 'tokens/trainable': 149435, 'epoch': '1.273'} |
|
64%|██████▎ | 326/512 [36:16<20:51, 6.73s/it]
64%|██████▍ | 327/512 [36:22<20:39, 6.70s/it]
{'loss': '0.0002422', 'grad_norm': '0.02509', 'learning_rate': '3.869e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.74', 'memory/max_allocated (GiB)': '33.74', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '39.82', 'tokens/total': 9889648, 'tokens/trainable': 149888, 'epoch': '1.277'} |
|
64%|██████▍ | 327/512 [36:22<20:39, 6.70s/it]
64%|██████▍ | 328/512 [36:29<20:30, 6.69s/it]
{'loss': '0.0005074', 'grad_norm': '0.04499', 'learning_rate': '3.842e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '34.24', 'tokens/total': 9919968, 'tokens/trainable': 150362, 'epoch': '1.281'} |
|
64%|██████▍ | 328/512 [36:29<20:30, 6.69s/it]
64%|██████▍ | 329/512 [36:36<20:19, 6.67s/it]
{'loss': '0.0001344', 'grad_norm': '0.01661', 'learning_rate': '3.815e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '32.21', 'tokens/total': 9950368, 'tokens/trainable': 150837, 'epoch': '1.285'} |
|
64%|██████▍ | 329/512 [36:36<20:19, 6.67s/it]
64%|██████▍ | 330/512 [36:42<20:11, 6.66s/it]
{'loss': '4.675e-05', 'grad_norm': '0.003686', 'learning_rate': '3.788e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '34.36', 'tokens/total': 9980688, 'tokens/trainable': 151273, 'epoch': '1.289'} |
|
64%|██████▍ | 330/512 [36:42<20:11, 6.66s/it]
65%|██████▍ | 331/512 [36:49<20:02, 6.65s/it]
{'loss': '2.638e-05', 'grad_norm': '0.001695', 'learning_rate': '3.761e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '37.63', 'tokens/total': 10011104, 'tokens/trainable': 151741, 'epoch': '1.293'} |
|
65%|██████▍ | 331/512 [36:49<20:02, 6.65s/it]
65%|██████▍ | 332/512 [36:55<19:53, 6.63s/it]
{'loss': '4.781e-05', 'grad_norm': '0.004727', 'learning_rate': '3.734e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '34.47', 'tokens/total': 10041536, 'tokens/trainable': 152191, 'epoch': '1.297'} |
|
65%|██████▍ | 332/512 [36:55<19:53, 6.63s/it]
65%|██████▌ | 333/512 [37:02<19:47, 6.63s/it]
{'loss': '0.0008173', 'grad_norm': '0.09555', 'learning_rate': '3.708e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '32.24', 'tokens/total': 10072112, 'tokens/trainable': 152630, 'epoch': '1.301'} |
|
65%|██████▌ | 333/512 [37:02<19:47, 6.63s/it]
65%|██████▌ | 334/512 [37:09<19:36, 6.61s/it]
{'loss': '0.000214', 'grad_norm': '0.02793', 'learning_rate': '3.681e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '33.85', 'tokens/total': 10102592, 'tokens/trainable': 153047, 'epoch': '1.305'} |
|
65%|██████▌ | 334/512 [37:09<19:36, 6.61s/it]
65%|██████▌ | 335/512 [37:15<19:29, 6.61s/it]
{'loss': '0.004335', 'grad_norm': '0.2731', 'learning_rate': '3.655e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '30.89', 'tokens/total': 10132736, 'tokens/trainable': 153477, 'epoch': '1.309'} |
|
65%|██████▌ | 335/512 [37:15<19:29, 6.61s/it]
66%|██████▌ | 336/512 [37:22<19:21, 6.60s/it]
{'loss': '9.929e-06', 'grad_norm': '0.002236', 'learning_rate': '3.628e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '33.72', 'tokens/total': 10162928, 'tokens/trainable': 153929, 'epoch': '1.312'} |
|
66%|██████▌ | 336/512 [37:22<19:21, 6.60s/it]
66%|██████▌ | 337/512 [37:28<19:20, 6.63s/it]
{'loss': '0.0006771', 'grad_norm': '0.05336', 'learning_rate': '3.602e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '35.34', 'tokens/total': 10193552, 'tokens/trainable': 154416, 'epoch': '1.316'} |
|
66%|██████▌ | 337/512 [37:28<19:20, 6.63s/it]
66%|██████▌ | 338/512 [37:35<19:14, 6.64s/it]
{'loss': '4.595e-05', 'grad_norm': '0.003405', 'learning_rate': '3.576e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '34.33', 'tokens/total': 10223808, 'tokens/trainable': 154865, 'epoch': '1.32'} |
|
66%|██████▌ | 338/512 [37:35<19:14, 6.64s/it]
66%|██████▌ | 339/512 [37:42<19:10, 6.65s/it]
{'loss': '1.35e-05', 'grad_norm': '0.0007159', 'learning_rate': '3.549e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '30.21', 'tokens/total': 10254272, 'tokens/trainable': 155290, 'epoch': '1.324'} |
|
66%|██████▌ | 339/512 [37:42<19:10, 6.65s/it]
66%|██████▋ | 340/512 [37:48<18:58, 6.62s/it]
{'loss': '7.454e-05', 'grad_norm': '0.01389', 'learning_rate': '3.523e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '31.6', 'tokens/total': 10284656, 'tokens/trainable': 155703, 'epoch': '1.328'} |
|
66%|██████▋ | 340/512 [37:48<18:58, 6.62s/it]
67%|██████▋ | 341/512 [37:55<18:54, 6.63s/it]
{'loss': '7.792e-05', 'grad_norm': '0.00568', 'learning_rate': '3.497e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '36.22', 'tokens/total': 10314976, 'tokens/trainable': 156193, 'epoch': '1.332'} |
|
67%|██████▋ | 341/512 [37:55<18:54, 6.63s/it]
67%|██████▋ | 342/512 [38:02<18:45, 6.62s/it]
{'loss': '0.0001705', 'grad_norm': '0.01501', 'learning_rate': '3.471e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.98', 'memory/max_allocated (GiB)': '33.98', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '34.09', 'tokens/total': 10345360, 'tokens/trainable': 156640, 'epoch': '1.336'} |
|
67%|██████▋ | 342/512 [38:02<18:45, 6.62s/it]
67%|██████▋ | 343/512 [38:08<18:37, 6.61s/it]
{'loss': '2.88e-05', 'grad_norm': '0.00322', 'learning_rate': '3.445e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.72', 'memory/max_allocated (GiB)': '33.72', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '34.14', 'tokens/total': 10375648, 'tokens/trainable': 157102, 'epoch': '1.34'} |
|
67%|██████▋ | 343/512 [38:08<18:37, 6.61s/it]
67%|██████▋ | 344/512 [38:15<18:31, 6.62s/it]
{'loss': '5.837e-05', 'grad_norm': '0.007129', 'learning_rate': '3.42e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '33.51', 'tokens/total': 10406144, 'tokens/trainable': 157544, 'epoch': '1.344'} |
|
67%|██████▋ | 344/512 [38:15<18:31, 6.62s/it]
67%|██████▋ | 345/512 [38:21<18:24, 6.61s/it]
{'loss': '0.007747', 'grad_norm': '0.3026', 'learning_rate': '3.394e-05', 'ppl': '1.008', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '35.6', 'tokens/total': 10436528, 'tokens/trainable': 157985, 'epoch': '1.348'} |
|
67%|██████▋ | 345/512 [38:21<18:24, 6.61s/it]
68%|██████▊ | 346/512 [38:28<18:16, 6.60s/it]
{'loss': '0.000125', 'grad_norm': '0.01498', 'learning_rate': '3.368e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '29.48', 'tokens/total': 10466864, 'tokens/trainable': 158402, 'epoch': '1.352'} |
|
68%|██████▊ | 346/512 [38:28<18:16, 6.60s/it]
68%|██████▊ | 347/512 [38:35<18:06, 6.59s/it]
{'loss': '2.681e-05', 'grad_norm': '0.002325', 'learning_rate': '3.343e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.73', 'memory/max_allocated (GiB)': '33.73', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '36.08', 'tokens/total': 10497008, 'tokens/trainable': 158860, 'epoch': '1.355'} |
|
68%|██████▊ | 347/512 [38:35<18:06, 6.59s/it]
68%|██████▊ | 348/512 [38:41<18:04, 6.61s/it]
{'loss': '0.0001693', 'grad_norm': '0.01576', 'learning_rate': '3.317e-05', 'ppl': '1', 'memory/max_active (GiB)': '34', 'memory/max_allocated (GiB)': '34', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '29.09', 'tokens/total': 10527728, 'tokens/trainable': 159282, 'epoch': '1.359'} |
|
68%|██████▊ | 348/512 [38:41<18:04, 6.61s/it]
68%|██████▊ | 349/512 [38:48<17:56, 6.60s/it]
{'loss': '0.001792', 'grad_norm': '0.2048', 'learning_rate': '3.292e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.76', 'memory/max_allocated (GiB)': '33.76', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '32.95', 'tokens/total': 10557856, 'tokens/trainable': 159732, 'epoch': '1.363'} |
|
68%|██████▊ | 349/512 [38:48<17:56, 6.60s/it]
68%|██████▊ | 350/512 [38:55<17:54, 6.63s/it]
{'loss': '1.568e-05', 'grad_norm': '0.0009161', 'learning_rate': '3.267e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.99', 'memory/max_allocated (GiB)': '33.99', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '35.68', 'tokens/total': 10588368, 'tokens/trainable': 160199, 'epoch': '1.367'} |
|
68%|██████▊ | 350/512 [38:55<17:54, 6.63s/it]
69%|██████▊ | 351/512 [39:01<17:47, 6.63s/it]
{'loss': '0.0003494', 'grad_norm': '0.05528', 'learning_rate': '3.242e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '33.04', 'tokens/total': 10618608, 'tokens/trainable': 160657, 'epoch': '1.371'} |
|
69%|██████▊ | 351/512 [39:01<17:47, 6.63s/it]
69%|██████▉ | 352/512 [39:08<17:39, 6.62s/it]
{'loss': '0.0002228', 'grad_norm': '0.01718', 'learning_rate': '3.217e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '34.98', 'tokens/total': 10648864, 'tokens/trainable': 161114, 'epoch': '1.375'} |
|
69%|██████▉ | 352/512 [39:08<17:39, 6.62s/it][2026-08-18 15:10:51,400] [INFO] [axolotl.core.trainers.base._save:828] [PID:12035] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-352 |
|
69%|██████▉ | 353/512 [39:16<18:50, 7.11s/it]
{'loss': '0.0001485', 'grad_norm': '0.01522', 'learning_rate': '3.192e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '29.27', 'tokens/total': 10679184, 'tokens/trainable': 161567, 'epoch': '1.379'} |
|
69%|██████▉ | 353/512 [39:16<18:50, 7.11s/it]
69%|██████▉ | 354/512 [39:23<18:19, 6.96s/it]
{'loss': '2.665e-05', 'grad_norm': '0.002392', 'learning_rate': '3.167e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '35.41', 'tokens/train_per_sec_per_gpu': '37.27', 'tokens/total': 10709648, 'tokens/trainable': 162032, 'epoch': '1.383'} |
|
69%|██████▉ | 354/512 [39:23<18:19, 6.96s/it]
69%|██████▉ | 355/512 [39:29<17:55, 6.85s/it]
{'loss': '0.0001098', 'grad_norm': '0.01253', 'learning_rate': '3.142e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '35.84', 'tokens/train_per_sec_per_gpu': '32.56', 'tokens/total': 10740128, 'tokens/trainable': 162460, 'epoch': '1.387'} |
|
69%|██████▉ | 355/512 [39:29<17:55, 6.85s/it]
70%|██████▉ | 356/512 [39:36<17:34, 6.76s/it]
{'loss': '2.361e-05', 'grad_norm': '0.001951', 'learning_rate': '3.117e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.61', 'memory/max_allocated (GiB)': '33.61', 'memory/device_reserved (GiB)': '35.84', 'tokens/train_per_sec_per_gpu': '28.61', 'tokens/total': 10770128, 'tokens/trainable': 162875, 'epoch': '1.391'} |
|
70%|██████▉ | 356/512 [39:36<17:34, 6.76s/it]
70%|██████▉ | 357/512 [39:42<17:20, 6.71s/it]
{'loss': '0.000105', 'grad_norm': '0.01397', 'learning_rate': '3.093e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.84', 'tokens/train_per_sec_per_gpu': '35.43', 'tokens/total': 10800432, 'tokens/trainable': 163329, 'epoch': '1.395'} |
|
70%|██████▉ | 357/512 [39:42<17:20, 6.71s/it]
70%|██████▉ | 358/512 [39:49<17:09, 6.69s/it]
{'loss': '0.0002698', 'grad_norm': '0.08449', 'learning_rate': '3.068e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '35.84', 'tokens/train_per_sec_per_gpu': '33.54', 'tokens/total': 10830592, 'tokens/trainable': 163778, 'epoch': '1.398'} |
|
70%|██████▉ | 358/512 [39:49<17:09, 6.69s/it]
70%|███████ | 359/512 [39:56<17:00, 6.67s/it]
{'loss': '0.001174', 'grad_norm': '0.07537', 'learning_rate': '3.044e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '35.84', 'tokens/train_per_sec_per_gpu': '32.17', 'tokens/total': 10861088, 'tokens/trainable': 164219, 'epoch': '1.402'} |
|
70%|███████ | 359/512 [39:56<17:00, 6.67s/it]
70%|███████ | 360/512 [40:02<16:50, 6.65s/it]
{'loss': '0.0002725', 'grad_norm': '0.02474', 'learning_rate': '3.02e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.84', 'tokens/train_per_sec_per_gpu': '34.66', 'tokens/total': 10891552, 'tokens/trainable': 164666, 'epoch': '1.406'} |
|
70%|███████ | 360/512 [40:02<16:50, 6.65s/it]
71%|███████ | 361/512 [40:09<16:40, 6.63s/it]
{'loss': '0.0167', 'grad_norm': '0.6778', 'learning_rate': '2.995e-05', 'ppl': '1.017', 'memory/max_active (GiB)': '33.77', 'memory/max_allocated (GiB)': '33.77', 'memory/device_reserved (GiB)': '35.84', 'tokens/train_per_sec_per_gpu': '37.15', 'tokens/total': 10921776, 'tokens/trainable': 165128, 'epoch': '1.41'} |
|
71%|███████ | 361/512 [40:09<16:40, 6.63s/it]
71%|███████ | 362/512 [40:15<16:35, 6.63s/it]
{'loss': '0.005559', 'grad_norm': '0.2795', 'learning_rate': '2.971e-05', 'ppl': '1.006', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '35.84', 'tokens/train_per_sec_per_gpu': '35.32', 'tokens/total': 10952288, 'tokens/trainable': 165614, 'epoch': '1.414'} |
|
71%|███████ | 362/512 [40:15<16:35, 6.63s/it]
71%|███████ | 363/512 [40:22<16:25, 6.61s/it]
{'loss': '0.00364', 'grad_norm': '0.1986', 'learning_rate': '2.947e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '35.84', 'tokens/train_per_sec_per_gpu': '34.59', 'tokens/total': 10982496, 'tokens/trainable': 166062, 'epoch': '1.418'} |
|
71%|███████ | 363/512 [40:22<16:25, 6.61s/it]
71%|███████ | 364/512 [40:29<16:19, 6.62s/it]
{'loss': '0.0001462', 'grad_norm': '0.02011', 'learning_rate': '2.924e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.84', 'tokens/train_per_sec_per_gpu': '34.55', 'tokens/total': 11012704, 'tokens/trainable': 166513, 'epoch': '1.422'} |
|
71%|███████ | 364/512 [40:29<16:19, 6.62s/it]
71%|███████▏ | 365/512 [40:35<16:13, 6.62s/it]
{'loss': '0.0006833', 'grad_norm': '0.09536', 'learning_rate': '2.9e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '35.84', 'tokens/train_per_sec_per_gpu': '36.77', 'tokens/total': 11043024, 'tokens/trainable': 167001, 'epoch': '1.426'} |
|
71%|███████▏ | 365/512 [40:35<16:13, 6.62s/it]
71%|███████▏ | 366/512 [40:42<16:07, 6.63s/it]
{'loss': '0.0002184', 'grad_norm': '0.01556', 'learning_rate': '2.876e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.84', 'tokens/train_per_sec_per_gpu': '30.62', 'tokens/total': 11073392, 'tokens/trainable': 167450, 'epoch': '1.43'} |
|
71%|███████▏ | 366/512 [40:42<16:07, 6.63s/it]
72%|███████▏ | 367/512 [40:49<16:01, 6.63s/it]
{'loss': '4.497e-05', 'grad_norm': '0.003135', 'learning_rate': '2.853e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '31.79', 'tokens/total': 11103936, 'tokens/trainable': 167895, 'epoch': '1.434'} |
|
72%|███████▏ | 367/512 [40:49<16:01, 6.63s/it]
72%|███████▏ | 368/512 [40:55<15:54, 6.63s/it]
{'loss': '0.0003686', 'grad_norm': '0.03942', 'learning_rate': '2.829e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '36', 'tokens/total': 11134256, 'tokens/trainable': 168338, 'epoch': '1.438'} |
|
72%|███████▏ | 368/512 [40:55<15:54, 6.63s/it]
72%|███████▏ | 369/512 [41:02<15:50, 6.64s/it]
{'loss': '0.001534', 'grad_norm': '0.07836', 'learning_rate': '2.806e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '32.61', 'tokens/total': 11164768, 'tokens/trainable': 168813, 'epoch': '1.441'} |
|
72%|███████▏ | 369/512 [41:02<15:50, 6.64s/it]
72%|███████▏ | 370/512 [41:08<15:43, 6.65s/it]
{'loss': '0.009904', 'grad_norm': '0.4711', 'learning_rate': '2.783e-05', 'ppl': '1.01', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '37.27', 'tokens/total': 11195136, 'tokens/trainable': 169315, 'epoch': '1.445'} |
|
72%|███████▏ | 370/512 [41:08<15:43, 6.65s/it]
72%|███████▏ | 371/512 [41:15<15:40, 6.67s/it]
{'loss': '0.0005743', 'grad_norm': '0.03302', 'learning_rate': '2.76e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '35.47', 'tokens/total': 11225680, 'tokens/trainable': 169756, 'epoch': '1.449'} |
|
72%|███████▏ | 371/512 [41:15<15:40, 6.67s/it]
73%|███████▎ | 372/512 [41:22<15:33, 6.67s/it]
{'loss': '0.0001973', 'grad_norm': '0.0127', 'learning_rate': '2.737e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '35.89', 'tokens/train_per_sec_per_gpu': '33.3', 'tokens/total': 11256112, 'tokens/trainable': 170222, 'epoch': '1.453'} |
|
73%|███████▎ | 372/512 [41:22<15:33, 6.67s/it]
73%|███████▎ | 373/512 [41:28<15:25, 6.66s/it]
{'loss': '8.283e-05', 'grad_norm': '0.005611', 'learning_rate': '2.714e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '35.89', 'tokens/train_per_sec_per_gpu': '33.6', 'tokens/total': 11286304, 'tokens/trainable': 170687, 'epoch': '1.457'} |
|
73%|███████▎ | 373/512 [41:28<15:25, 6.66s/it]
73%|███████▎ | 374/512 [41:35<15:17, 6.65s/it]
{'loss': '0.001198', 'grad_norm': '0.1377', 'learning_rate': '2.691e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '35.89', 'tokens/train_per_sec_per_gpu': '32.43', 'tokens/total': 11316800, 'tokens/trainable': 171163, 'epoch': '1.461'} |
|
73%|███████▎ | 374/512 [41:35<15:17, 6.65s/it]
73%|███████▎ | 375/512 [41:42<15:07, 6.62s/it]
{'loss': '0.007628', 'grad_norm': '0.4246', 'learning_rate': '2.668e-05', 'ppl': '1.008', 'memory/max_active (GiB)': '33.72', 'memory/max_allocated (GiB)': '33.72', 'memory/device_reserved (GiB)': '35.89', 'tokens/train_per_sec_per_gpu': '34.48', 'tokens/total': 11346944, 'tokens/trainable': 171626, 'epoch': '1.465'} |
|
73%|███████▎ | 375/512 [41:42<15:07, 6.62s/it]
73%|███████▎ | 376/512 [41:48<14:59, 6.62s/it]
{'loss': '0.0001166', 'grad_norm': '0.008275', 'learning_rate': '2.646e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.89', 'tokens/train_per_sec_per_gpu': '28.49', 'tokens/total': 11377376, 'tokens/trainable': 172043, 'epoch': '1.469'} |
|
73%|███████▎ | 376/512 [41:48<14:59, 6.62s/it]
74%|███████▎ | 377/512 [41:55<14:53, 6.62s/it]
{'loss': '0.0001835', 'grad_norm': '0.01685', 'learning_rate': '2.624e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.89', 'tokens/train_per_sec_per_gpu': '36.55', 'tokens/total': 11407680, 'tokens/trainable': 172519, 'epoch': '1.473'} |
|
74%|███████▎ | 377/512 [41:55<14:53, 6.62s/it]
74%|███████▍ | 378/512 [42:02<14:45, 6.61s/it]
{'loss': '0.003047', 'grad_norm': '0.3305', 'learning_rate': '2.601e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.71', 'memory/max_allocated (GiB)': '33.71', 'memory/device_reserved (GiB)': '35.89', 'tokens/train_per_sec_per_gpu': '34.88', 'tokens/total': 11437712, 'tokens/trainable': 172977, 'epoch': '1.477'} |
|
74%|███████▍ | 378/512 [42:02<14:45, 6.61s/it]
74%|███████▍ | 379/512 [42:08<14:38, 6.61s/it]
{'loss': '0.001019', 'grad_norm': '0.1227', 'learning_rate': '2.579e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '35.89', 'tokens/train_per_sec_per_gpu': '36.22', 'tokens/total': 11468176, 'tokens/trainable': 173461, 'epoch': '1.48'} |
|
74%|███████▍ | 379/512 [42:08<14:38, 6.61s/it]
74%|███████▍ | 380/512 [42:15<14:32, 6.61s/it]
{'loss': '4.365e-05', 'grad_norm': '0.001798', 'learning_rate': '2.557e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.73', 'memory/max_allocated (GiB)': '33.73', 'memory/device_reserved (GiB)': '35.89', 'tokens/train_per_sec_per_gpu': '30.87', 'tokens/total': 11498352, 'tokens/trainable': 173926, 'epoch': '1.484'} |
|
74%|███████▍ | 380/512 [42:15<14:32, 6.61s/it]
74%|███████▍ | 381/512 [42:21<14:27, 6.62s/it]
{'loss': '0.005217', 'grad_norm': '0.2067', 'learning_rate': '2.535e-05', 'ppl': '1.005', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '35.89', 'tokens/train_per_sec_per_gpu': '34.61', 'tokens/total': 11528768, 'tokens/trainable': 174397, 'epoch': '1.488'} |
|
74%|███████▍ | 381/512 [42:21<14:27, 6.62s/it]
75%|███████▍ | 382/512 [42:28<14:20, 6.62s/it]
{'loss': '0.001172', 'grad_norm': '0.08067', 'learning_rate': '2.513e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '35.89', 'tokens/train_per_sec_per_gpu': '33.27', 'tokens/total': 11558976, 'tokens/trainable': 174852, 'epoch': '1.492'} |
|
75%|███████▍ | 382/512 [42:28<14:20, 6.62s/it]
75%|███████▍ | 383/512 [42:35<14:15, 6.63s/it]
{'loss': '0.0002196', 'grad_norm': '0.02767', 'learning_rate': '2.492e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '35.89', 'tokens/train_per_sec_per_gpu': '31.88', 'tokens/total': 11589520, 'tokens/trainable': 175293, 'epoch': '1.496'} |
|
75%|███████▍ | 383/512 [42:35<14:15, 6.63s/it]
75%|███████▌ | 384/512 [42:41<14:09, 6.64s/it]
{'loss': '0.001598', 'grad_norm': '0.1287', 'learning_rate': '2.47e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '35.89', 'tokens/train_per_sec_per_gpu': '32.87', 'tokens/total': 11619760, 'tokens/trainable': 175731, 'epoch': '1.5'} |
|
75%|███████▌ | 384/512 [42:41<14:09, 6.64s/it][2026-08-18 15:14:24,959] [INFO] [axolotl.core.trainers.base._save:828] [PID:12035] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-384 |
|
75%|███████▌ | 385/512 [42:50<15:27, 7.30s/it]
{'loss': '0.0005179', 'grad_norm': '0.03734', 'learning_rate': '2.449e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.89', 'tokens/train_per_sec_per_gpu': '35.62', 'tokens/total': 11650192, 'tokens/trainable': 176215, 'epoch': '1.504'} |
|
75%|███████▌ | 385/512 [42:50<15:27, 7.30s/it]
75%|███████▌ | 386/512 [42:57<14:53, 7.09s/it]
{'loss': '0.0005837', 'grad_norm': '0.06761', 'learning_rate': '2.428e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '34.8', 'tokens/total': 11680672, 'tokens/trainable': 176663, 'epoch': '1.508'} |
|
75%|███████▌ | 386/512 [42:57<14:53, 7.09s/it]
76%|███████▌ | 387/512 [43:03<14:29, 6.96s/it]
{'loss': '0.0005241', 'grad_norm': '0.05018', 'learning_rate': '2.406e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '33.11', 'tokens/total': 11710880, 'tokens/trainable': 177127, 'epoch': '1.512'} |
|
76%|███████▌ | 387/512 [43:03<14:29, 6.96s/it]
76%|███████▌ | 388/512 [43:10<14:10, 6.86s/it]
{'loss': '0.001148', 'grad_norm': '0.08329', 'learning_rate': '2.385e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '36.56', 'tokens/total': 11741408, 'tokens/trainable': 177613, 'epoch': '1.516'} |
|
76%|███████▌ | 388/512 [43:10<14:10, 6.86s/it]
76%|███████▌ | 389/512 [43:17<13:54, 6.78s/it]
{'loss': '0.000309', 'grad_norm': '0.02066', 'learning_rate': '2.365e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '33.47', 'tokens/total': 11771712, 'tokens/trainable': 178062, 'epoch': '1.52'} |
|
76%|███████▌ | 389/512 [43:17<13:54, 6.78s/it]
76%|███████▌ | 390/512 [43:23<13:41, 6.74s/it]
{'loss': '0.00147', 'grad_norm': '0.1384', 'learning_rate': '2.344e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '36.96', 'tokens/total': 11801872, 'tokens/trainable': 178535, 'epoch': '1.523'} |
|
76%|███████▌ | 390/512 [43:23<13:41, 6.74s/it]
76%|███████▋ | 391/512 [43:30<13:32, 6.71s/it]
{'loss': '0.0007466', 'grad_norm': '0.06139', 'learning_rate': '2.323e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '34.42', 'tokens/total': 11832480, 'tokens/trainable': 179004, 'epoch': '1.527'} |
|
76%|███████▋ | 391/512 [43:30<13:32, 6.71s/it]
77%|███████▋ | 392/512 [43:36<13:20, 6.67s/it]
{'loss': '5.849e-05', 'grad_norm': '0.003179', 'learning_rate': '2.303e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '33.49', 'tokens/total': 11862672, 'tokens/trainable': 179461, 'epoch': '1.531'} |
|
77%|███████▋ | 392/512 [43:36<13:20, 6.67s/it]
77%|███████▋ | 393/512 [43:43<13:09, 6.64s/it]
{'loss': '0.0002381', 'grad_norm': '0.02178', 'learning_rate': '2.282e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '33.22', 'tokens/total': 11893024, 'tokens/trainable': 179893, 'epoch': '1.535'} |
|
77%|███████▋ | 393/512 [43:43<13:09, 6.64s/it]
77%|███████▋ | 394/512 [43:50<13:01, 6.63s/it]
{'loss': '0.0007996', 'grad_norm': '0.1055', 'learning_rate': '2.262e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '34.41', 'tokens/total': 11923328, 'tokens/trainable': 180344, 'epoch': '1.539'} |
|
77%|███████▋ | 394/512 [43:50<13:01, 6.63s/it]
77%|███████▋ | 395/512 [43:56<12:52, 6.60s/it]
{'loss': '5.368e-05', 'grad_norm': '0.003161', 'learning_rate': '2.242e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '32.21', 'tokens/total': 11953792, 'tokens/trainable': 180757, 'epoch': '1.543'} |
|
77%|███████▋ | 395/512 [43:56<12:52, 6.60s/it]
77%|███████▋ | 396/512 [44:03<12:46, 6.61s/it]
{'loss': '0.0009188', 'grad_norm': '0.05269', 'learning_rate': '2.222e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '35.64', 'tokens/total': 11984208, 'tokens/trainable': 181228, 'epoch': '1.547'} |
|
77%|███████▋ | 396/512 [44:03<12:46, 6.61s/it]
78%|███████▊ | 397/512 [44:09<12:40, 6.61s/it]
{'loss': '3.819e-05', 'grad_norm': '0.001785', 'learning_rate': '2.202e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '33.66', 'tokens/total': 12014608, 'tokens/trainable': 181716, 'epoch': '1.551'} |
|
78%|███████▊ | 397/512 [44:09<12:40, 6.61s/it]
78%|███████▊ | 398/512 [44:16<12:32, 6.60s/it]
{'loss': '0.002452', 'grad_norm': '0.1685', 'learning_rate': '2.183e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '36.01', 'tokens/total': 12045088, 'tokens/trainable': 182152, 'epoch': '1.555'} |
|
78%|███████▊ | 398/512 [44:16<12:32, 6.60s/it]
78%|███████▊ | 399/512 [44:23<12:26, 6.61s/it]
{'loss': '0.001032', 'grad_norm': '0.1164', 'learning_rate': '2.163e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.79', 'memory/max_allocated (GiB)': '33.79', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '35.37', 'tokens/total': 12075280, 'tokens/trainable': 182623, 'epoch': '1.559'} |
|
78%|███████▊ | 399/512 [44:23<12:26, 6.61s/it]
78%|███████▊ | 400/512 [44:29<12:20, 6.61s/it]
{'loss': '7.536e-05', 'grad_norm': '0.00444', 'learning_rate': '2.144e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '26.92', 'tokens/total': 12105792, 'tokens/trainable': 183039, 'epoch': '1.562'} |
|
78%|███████▊ | 400/512 [44:29<12:20, 6.61s/it]
78%|███████▊ | 401/512 [44:36<12:12, 6.60s/it]
{'loss': '0.000861', 'grad_norm': '0.06607', 'learning_rate': '2.124e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.71', 'memory/max_allocated (GiB)': '33.71', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '32.2', 'tokens/total': 12135680, 'tokens/trainable': 183487, 'epoch': '1.566'} |
|
78%|███████▊ | 401/512 [44:36<12:12, 6.60s/it]
79%|███████▊ | 402/512 [44:43<12:09, 6.63s/it]
{'loss': '6.076e-05', 'grad_norm': '0.0037', 'learning_rate': '2.105e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '39.03', 'tokens/total': 12165920, 'tokens/trainable': 183987, 'epoch': '1.57'} |
|
79%|███████▊ | 402/512 [44:43<12:09, 6.63s/it]
79%|███████▊ | 403/512 [44:49<12:03, 6.64s/it]
{'loss': '0.0001372', 'grad_norm': '0.01057', 'learning_rate': '2.086e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '32.86', 'tokens/total': 12196080, 'tokens/trainable': 184459, 'epoch': '1.574'} |
|
79%|███████▊ | 403/512 [44:49<12:03, 6.64s/it]
79%|███████▉ | 404/512 [44:56<11:56, 6.63s/it]
{'loss': '0.003382', 'grad_norm': '0.3721', 'learning_rate': '2.067e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '31.95', 'tokens/total': 12226128, 'tokens/trainable': 184887, 'epoch': '1.578'} |
|
79%|███████▉ | 404/512 [44:56<11:56, 6.63s/it]
79%|███████▉ | 405/512 [45:02<11:48, 6.62s/it]
{'loss': '0.0008268', 'grad_norm': '0.08975', 'learning_rate': '2.049e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.72', 'memory/max_allocated (GiB)': '33.72', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '29.59', 'tokens/total': 12256176, 'tokens/trainable': 185320, 'epoch': '1.582'} |
|
79%|███████▉ | 405/512 [45:02<11:48, 6.62s/it]
79%|███████▉ | 406/512 [45:09<11:42, 6.63s/it]
{'loss': '0.01279', 'grad_norm': '0.4966', 'learning_rate': '2.03e-05', 'ppl': '1.013', 'memory/max_active (GiB)': '33.79', 'memory/max_allocated (GiB)': '33.79', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '31.77', 'tokens/total': 12286416, 'tokens/trainable': 185774, 'epoch': '1.586'} |
|
79%|███████▉ | 406/512 [45:09<11:42, 6.63s/it]
79%|███████▉ | 407/512 [45:16<11:34, 6.62s/it]
{'loss': '3.963e-05', 'grad_norm': '0.001786', 'learning_rate': '2.012e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.88', 'tokens/train_per_sec_per_gpu': '37.15', 'tokens/total': 12316720, 'tokens/trainable': 186233, 'epoch': '1.59'} |
|
79%|███████▉ | 407/512 [45:16<11:34, 6.62s/it]
80%|███████▉ | 408/512 [45:22<11:28, 6.62s/it]
{'loss': '6.553e-05', 'grad_norm': '0.006782', 'learning_rate': '1.993e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '35.96', 'tokens/train_per_sec_per_gpu': '33.14', 'tokens/total': 12347072, 'tokens/trainable': 186671, 'epoch': '1.594'} |
|
80%|███████▉ | 408/512 [45:22<11:28, 6.62s/it]
80%|███████▉ | 409/512 [45:29<11:23, 6.63s/it]
{'loss': '0.0002536', 'grad_norm': '0.02035', 'learning_rate': '1.975e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '35.96', 'tokens/train_per_sec_per_gpu': '34.29', 'tokens/total': 12377504, 'tokens/trainable': 187102, 'epoch': '1.598'} |
|
80%|███████▉ | 409/512 [45:29<11:23, 6.63s/it]
80%|████████ | 410/512 [45:36<11:17, 6.64s/it]
{'loss': '5.506e-05', 'grad_norm': '0.003249', 'learning_rate': '1.957e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.96', 'tokens/train_per_sec_per_gpu': '35.25', 'tokens/total': 12408048, 'tokens/trainable': 187549, 'epoch': '1.602'} |
|
80%|████████ | 410/512 [45:36<11:17, 6.64s/it]
80%|████████ | 411/512 [45:42<11:10, 6.64s/it]
{'loss': '0.0007389', 'grad_norm': '0.1669', 'learning_rate': '1.94e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.96', 'tokens/train_per_sec_per_gpu': '34.61', 'tokens/total': 12438256, 'tokens/trainable': 187990, 'epoch': '1.605'} |
|
80%|████████ | 411/512 [45:42<11:10, 6.64s/it]
80%|████████ | 412/512 [45:48<10:46, 6.47s/it]
{'loss': '0.0001102', 'grad_norm': '0.005521', 'learning_rate': '1.922e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '35.96', 'tokens/train_per_sec_per_gpu': '39.21', 'tokens/total': 12466608, 'tokens/trainable': 188426, 'epoch': '1.609'} |
|
80%|████████ | 412/512 [45:48<10:46, 6.47s/it]
81%|████████ | 413/512 [45:55<10:46, 6.53s/it]
{'loss': '0.001613', 'grad_norm': '0.1134', 'learning_rate': '1.904e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.96', 'tokens/train_per_sec_per_gpu': '31.96', 'tokens/total': 12497136, 'tokens/trainable': 188877, 'epoch': '1.613'} |
|
81%|████████ | 413/512 [45:55<10:46, 6.53s/it]
81%|████████ | 414/512 [46:02<10:40, 6.54s/it]
{'loss': '0.0002309', 'grad_norm': '0.09549', 'learning_rate': '1.887e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '35.96', 'tokens/train_per_sec_per_gpu': '37.84', 'tokens/total': 12527120, 'tokens/trainable': 189346, 'epoch': '1.617'} |
|
81%|████████ | 414/512 [46:02<10:40, 6.54s/it]
81%|████████ | 415/512 [46:08<10:34, 6.54s/it]
{'loss': '4.779e-05', 'grad_norm': '0.003373', 'learning_rate': '1.87e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.77', 'memory/max_allocated (GiB)': '33.77', 'memory/device_reserved (GiB)': '35.96', 'tokens/train_per_sec_per_gpu': '32.34', 'tokens/total': 12557248, 'tokens/trainable': 189759, 'epoch': '1.621'} |
|
81%|████████ | 415/512 [46:08<10:34, 6.54s/it]
81%|████████▏ | 416/512 [46:15<10:30, 6.57s/it]
{'loss': '4.39e-05', 'grad_norm': '0.003136', 'learning_rate': '1.853e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.96', 'tokens/train_per_sec_per_gpu': '34.5', 'tokens/total': 12587728, 'tokens/trainable': 190225, 'epoch': '1.625'} |
|
81%|████████▏ | 416/512 [46:15<10:30, 6.57s/it][2026-08-18 15:17:58,362] [INFO] [axolotl.core.trainers.base._save:828] [PID:12035] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-416 |
|
81%|████████▏ | 417/512 [46:23<11:16, 7.12s/it]
{'loss': '5.062e-05', 'grad_norm': '0.00255', 'learning_rate': '1.836e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '35.96', 'tokens/train_per_sec_per_gpu': '35.49', 'tokens/total': 12618160, 'tokens/trainable': 190717, 'epoch': '1.629'} |
|
81%|████████▏ | 417/512 [46:23<11:16, 7.12s/it]
82%|████████▏ | 418/512 [46:30<10:53, 6.95s/it]
{'loss': '9.37e-05', 'grad_norm': '0.006293', 'learning_rate': '1.819e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '35.81', 'tokens/train_per_sec_per_gpu': '31.28', 'tokens/total': 12648672, 'tokens/trainable': 191135, 'epoch': '1.633'} |
|
82%|████████▏ | 418/512 [46:30<10:53, 6.95s/it]
82%|████████▏ | 419/512 [46:36<10:36, 6.84s/it]
{'loss': '0.002148', 'grad_norm': '0.1299', 'learning_rate': '1.802e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '35.81', 'tokens/train_per_sec_per_gpu': '31.86', 'tokens/total': 12678944, 'tokens/trainable': 191559, 'epoch': '1.637'} |
|
82%|████████▏ | 419/512 [46:36<10:36, 6.84s/it]
82%|████████▏ | 420/512 [46:43<10:23, 6.78s/it]
{'loss': '0.0001932', 'grad_norm': '0.01716', 'learning_rate': '1.786e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '35.81', 'tokens/train_per_sec_per_gpu': '36.98', 'tokens/total': 12709408, 'tokens/trainable': 192033, 'epoch': '1.641'} |
|
82%|████████▏ | 420/512 [46:43<10:23, 6.78s/it]
82%|████████▏ | 421/512 [46:50<10:13, 6.74s/it]
{'loss': '5.247e-05', 'grad_norm': '0.004589', 'learning_rate': '1.77e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.81', 'tokens/train_per_sec_per_gpu': '31.53', 'tokens/total': 12739632, 'tokens/trainable': 192482, 'epoch': '1.645'} |
|
82%|████████▏ | 421/512 [46:50<10:13, 6.74s/it]
82%|████████▏ | 422/512 [46:56<10:04, 6.71s/it]
{'loss': '0.0002731', 'grad_norm': '0.04829', 'learning_rate': '1.753e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '35.83', 'tokens/train_per_sec_per_gpu': '33', 'tokens/total': 12770224, 'tokens/trainable': 192937, 'epoch': '1.648'} |
|
82%|████████▏ | 422/512 [46:56<10:04, 6.71s/it]
83%|████████▎ | 423/512 [47:03<09:56, 6.70s/it]
{'loss': '0.0003945', 'grad_norm': '0.03392', 'learning_rate': '1.737e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '35.83', 'tokens/train_per_sec_per_gpu': '37.81', 'tokens/total': 12800640, 'tokens/trainable': 193444, 'epoch': '1.652'} |
|
83%|████████▎ | 423/512 [47:03<09:56, 6.70s/it]
83%|████████▎ | 424/512 [47:09<09:47, 6.68s/it]
{'loss': '0.0001652', 'grad_norm': '0.01164', 'learning_rate': '1.722e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '35.83', 'tokens/train_per_sec_per_gpu': '36.92', 'tokens/total': 12831104, 'tokens/trainable': 193925, 'epoch': '1.656'} |
|
83%|████████▎ | 424/512 [47:09<09:47, 6.68s/it]
83%|████████▎ | 425/512 [47:16<09:39, 6.66s/it]
{'loss': '0.0008455', 'grad_norm': '0.06535', 'learning_rate': '1.706e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.79', 'memory/max_allocated (GiB)': '33.79', 'memory/device_reserved (GiB)': '35.83', 'tokens/train_per_sec_per_gpu': '29.68', 'tokens/total': 12861296, 'tokens/trainable': 194342, 'epoch': '1.66'} |
|
83%|████████▎ | 425/512 [47:16<09:39, 6.66s/it]
83%|████████▎ | 426/512 [47:23<09:32, 6.65s/it]
{'loss': '0.0007431', 'grad_norm': '0.072', 'learning_rate': '1.69e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.79', 'memory/max_allocated (GiB)': '33.79', 'memory/device_reserved (GiB)': '35.83', 'tokens/train_per_sec_per_gpu': '35.4', 'tokens/total': 12891504, 'tokens/trainable': 194830, 'epoch': '1.664'} |
|
83%|████████▎ | 426/512 [47:23<09:32, 6.65s/it]
83%|████████▎ | 427/512 [47:29<09:25, 6.65s/it]
{'loss': '0.0002793', 'grad_norm': '0.03094', 'learning_rate': '1.675e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.83', 'tokens/train_per_sec_per_gpu': '34.47', 'tokens/total': 12921840, 'tokens/trainable': 195296, 'epoch': '1.668'} |
|
83%|████████▎ | 427/512 [47:29<09:25, 6.65s/it]
84%|████████▎ | 428/512 [47:36<09:17, 6.64s/it]
{'loss': '4.714e-05', 'grad_norm': '0.00303', 'learning_rate': '1.66e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.83', 'tokens/train_per_sec_per_gpu': '32.33', 'tokens/total': 12952224, 'tokens/trainable': 195714, 'epoch': '1.672'} |
|
84%|████████▎ | 428/512 [47:36<09:17, 6.64s/it]
84%|████████▍ | 429/512 [47:43<09:09, 6.62s/it]
{'loss': '0.0008514', 'grad_norm': '0.05571', 'learning_rate': '1.645e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.75', 'memory/max_allocated (GiB)': '33.75', 'memory/device_reserved (GiB)': '35.83', 'tokens/train_per_sec_per_gpu': '33.31', 'tokens/total': 12982416, 'tokens/trainable': 196179, 'epoch': '1.676'} |
|
84%|████████▍ | 429/512 [47:43<09:09, 6.62s/it]
84%|████████▍ | 430/512 [47:49<09:03, 6.63s/it]
{'loss': '0.0002428', 'grad_norm': '0.0149', 'learning_rate': '1.63e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.83', 'tokens/train_per_sec_per_gpu': '32.82', 'tokens/total': 13012800, 'tokens/trainable': 196660, 'epoch': '1.68'} |
|
84%|████████▍ | 430/512 [47:49<09:03, 6.63s/it]
84%|████████▍ | 431/512 [47:56<08:59, 6.66s/it]
{'loss': '5.273e-05', 'grad_norm': '0.003593', 'learning_rate': '1.615e-05', 'ppl': '1', 'memory/max_active (GiB)': '34.02', 'memory/max_allocated (GiB)': '34.02', 'memory/device_reserved (GiB)': '36.35', 'tokens/train_per_sec_per_gpu': '31.51', 'tokens/total': 13043392, 'tokens/trainable': 197091, 'epoch': '1.684'} |
|
84%|████████▍ | 431/512 [47:56<08:59, 6.66s/it]
84%|████████▍ | 432/512 [48:03<08:52, 6.66s/it]
{'loss': '0.0003876', 'grad_norm': '0.03395', 'learning_rate': '1.6e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '36.35', 'tokens/train_per_sec_per_gpu': '39.07', 'tokens/total': 13073984, 'tokens/trainable': 197589, 'epoch': '1.688'} |
|
84%|████████▍ | 432/512 [48:03<08:52, 6.66s/it]
85%|████████▍ | 433/512 [48:09<08:45, 6.65s/it]
{'loss': '0.0007863', 'grad_norm': '0.06874', 'learning_rate': '1.586e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.95', 'memory/max_allocated (GiB)': '33.95', 'memory/device_reserved (GiB)': '36.8', 'tokens/train_per_sec_per_gpu': '33.15', 'tokens/total': 13104512, 'tokens/trainable': 198047, 'epoch': '1.691'} |
|
85%|████████▍ | 433/512 [48:09<08:45, 6.65s/it]
85%|████████▍ | 434/512 [48:16<08:38, 6.65s/it]
{'loss': '0.0004497', 'grad_norm': '0.04059', 'learning_rate': '1.572e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '36.8', 'tokens/train_per_sec_per_gpu': '32.89', 'tokens/total': 13134656, 'tokens/trainable': 198465, 'epoch': '1.695'} |
|
85%|████████▍ | 434/512 [48:16<08:38, 6.65s/it]
85%|████████▍ | 435/512 [48:22<08:31, 6.64s/it]
{'loss': '0.001607', 'grad_norm': '0.1532', 'learning_rate': '1.558e-05', 'ppl': '1.002', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '36.8', 'tokens/train_per_sec_per_gpu': '34.75', 'tokens/total': 13165248, 'tokens/trainable': 198937, 'epoch': '1.699'} |
|
85%|████████▍ | 435/512 [48:22<08:31, 6.64s/it]
85%|████████▌ | 436/512 [48:29<08:24, 6.64s/it]
{'loss': '0.0005046', 'grad_norm': '0.03812', 'learning_rate': '1.544e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '36.8', 'tokens/train_per_sec_per_gpu': '33.7', 'tokens/total': 13195648, 'tokens/trainable': 199385, 'epoch': '1.703'} |
|
85%|████████▌ | 436/512 [48:29<08:24, 6.64s/it]
85%|████████▌ | 437/512 [48:36<08:17, 6.63s/it]
{'loss': '0.008203', 'grad_norm': '0.3263', 'learning_rate': '1.53e-05', 'ppl': '1.008', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.8', 'tokens/train_per_sec_per_gpu': '34.78', 'tokens/total': 13226176, 'tokens/trainable': 199869, 'epoch': '1.707'} |
|
85%|████████▌ | 437/512 [48:36<08:17, 6.63s/it]
86%|████████▌ | 438/512 [48:42<08:11, 6.65s/it]
{'loss': '1.893e-05', 'grad_norm': '0.00103', 'learning_rate': '1.516e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '36.8', 'tokens/train_per_sec_per_gpu': '35.38', 'tokens/total': 13256480, 'tokens/trainable': 200377, 'epoch': '1.711'} |
|
86%|████████▌ | 438/512 [48:42<08:11, 6.65s/it]
86%|████████▌ | 439/512 [48:49<08:03, 6.62s/it]
{'loss': '0.0001817', 'grad_norm': '0.02155', 'learning_rate': '1.503e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.77', 'memory/max_allocated (GiB)': '33.77', 'memory/device_reserved (GiB)': '36.8', 'tokens/train_per_sec_per_gpu': '33.25', 'tokens/total': 13286528, 'tokens/trainable': 200824, 'epoch': '1.715'} |
|
86%|████████▌ | 439/512 [48:49<08:03, 6.62s/it]
86%|████████▌ | 440/512 [48:56<07:57, 6.63s/it]
{'loss': '0.0003251', 'grad_norm': '0.02102', 'learning_rate': '1.49e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '36.8', 'tokens/train_per_sec_per_gpu': '35.51', 'tokens/total': 13317024, 'tokens/trainable': 201297, 'epoch': '1.719'} |
|
86%|████████▌ | 440/512 [48:56<07:57, 6.63s/it]
86%|████████▌ | 441/512 [49:02<07:50, 6.63s/it]
{'loss': '0.0001027', 'grad_norm': '0.007309', 'learning_rate': '1.477e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '36.8', 'tokens/train_per_sec_per_gpu': '34.67', 'tokens/total': 13347440, 'tokens/trainable': 201764, 'epoch': '1.723'} |
|
86%|████████▌ | 441/512 [49:02<07:50, 6.63s/it]
86%|████████▋ | 442/512 [49:09<07:43, 6.63s/it]
{'loss': '0.0005666', 'grad_norm': '0.04583', 'learning_rate': '1.464e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '36.8', 'tokens/train_per_sec_per_gpu': '30.84', 'tokens/total': 13377664, 'tokens/trainable': 202204, 'epoch': '1.727'} |
|
86%|████████▋ | 442/512 [49:09<07:43, 6.63s/it]
87%|████████▋ | 443/512 [49:16<07:37, 6.63s/it]
{'loss': '0.004353', 'grad_norm': '0.2436', 'learning_rate': '1.451e-05', 'ppl': '1.004', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '36.8', 'tokens/train_per_sec_per_gpu': '36.59', 'tokens/total': 13408080, 'tokens/trainable': 202642, 'epoch': '1.73'} |
|
87%|████████▋ | 443/512 [49:16<07:37, 6.63s/it]
87%|████████▋ | 444/512 [49:22<07:28, 6.59s/it]
{'loss': '0.000228', 'grad_norm': '0.01532', 'learning_rate': '1.438e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.64', 'memory/max_allocated (GiB)': '33.64', 'memory/device_reserved (GiB)': '36.8', 'tokens/train_per_sec_per_gpu': '31.22', 'tokens/total': 13437824, 'tokens/trainable': 203050, 'epoch': '1.734'} |
|
87%|████████▋ | 444/512 [49:22<07:28, 6.59s/it]
87%|████████▋ | 445/512 [49:29<07:22, 6.60s/it]
{'loss': '2.628e-05', 'grad_norm': '0.001495', 'learning_rate': '1.426e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.77', 'memory/max_allocated (GiB)': '33.77', 'memory/device_reserved (GiB)': '36.8', 'tokens/train_per_sec_per_gpu': '34.63', 'tokens/total': 13468032, 'tokens/trainable': 203509, 'epoch': '1.738'} |
|
87%|████████▋ | 445/512 [49:29<07:22, 6.60s/it]
87%|████████▋ | 446/512 [49:35<07:16, 6.61s/it]
{'loss': '6.653e-05', 'grad_norm': '0.009212', 'learning_rate': '1.414e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '36.8', 'tokens/train_per_sec_per_gpu': '35.78', 'tokens/total': 13498096, 'tokens/trainable': 203976, 'epoch': '1.742'} |
|
87%|████████▋ | 446/512 [49:35<07:16, 6.61s/it]
87%|████████▋ | 447/512 [49:42<07:09, 6.60s/it]
{'loss': '0.001197', 'grad_norm': '0.1536', 'learning_rate': '1.402e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '36.8', 'tokens/train_per_sec_per_gpu': '37.97', 'tokens/total': 13528288, 'tokens/trainable': 204412, 'epoch': '1.746'} |
|
87%|████████▋ | 447/512 [49:42<07:09, 6.60s/it]
88%|████████▊ | 448/512 [49:48<07:02, 6.60s/it]
{'loss': '9.093e-05', 'grad_norm': '0.008475', 'learning_rate': '1.39e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '36.8', 'tokens/train_per_sec_per_gpu': '36.02', 'tokens/total': 13558752, 'tokens/trainable': 204868, 'epoch': '1.75'} |
|
88%|████████▊ | 448/512 [49:48<07:02, 6.60s/it][2026-08-18 15:21:32,137] [INFO] [axolotl.core.trainers.base._save:828] [PID:12035] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-448 |
|
88%|████████▊ | 449/512 [49:57<07:28, 7.13s/it]
{'loss': '0.0001275', 'grad_norm': '0.01541', 'learning_rate': '1.378e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.93', 'memory/max_allocated (GiB)': '33.93', 'memory/device_reserved (GiB)': '36.8', 'tokens/train_per_sec_per_gpu': '34.11', 'tokens/total': 13589136, 'tokens/trainable': 205338, 'epoch': '1.754'} |
|
88%|████████▊ | 449/512 [49:57<07:28, 7.13s/it]
88%|████████▊ | 450/512 [50:03<07:12, 6.98s/it]
{'loss': '0.0008909', 'grad_norm': '0.1296', 'learning_rate': '1.367e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.7', 'tokens/train_per_sec_per_gpu': '31.08', 'tokens/total': 13619472, 'tokens/trainable': 205764, 'epoch': '1.758'} |
|
88%|████████▊ | 450/512 [50:03<07:12, 6.98s/it]
88%|████████▊ | 451/512 [50:10<06:59, 6.88s/it]
{'loss': '0.0001473', 'grad_norm': '0.01026', 'learning_rate': '1.355e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.7', 'tokens/train_per_sec_per_gpu': '32.08', 'tokens/total': 13649872, 'tokens/trainable': 206201, 'epoch': '1.762'} |
|
88%|████████▊ | 451/512 [50:10<06:59, 6.88s/it]
88%|████████▊ | 452/512 [50:17<06:47, 6.80s/it]
{'loss': '9.224e-05', 'grad_norm': '0.006176', 'learning_rate': '1.344e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.7', 'tokens/train_per_sec_per_gpu': '35.89', 'tokens/total': 13680064, 'tokens/trainable': 206657, 'epoch': '1.766'} |
|
88%|████████▊ | 452/512 [50:17<06:47, 6.80s/it]
88%|████████▊ | 453/512 [50:23<06:38, 6.76s/it]
{'loss': '0.0005743', 'grad_norm': '0.06065', 'learning_rate': '1.333e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.98', 'memory/max_allocated (GiB)': '33.98', 'memory/device_reserved (GiB)': '35.7', 'tokens/train_per_sec_per_gpu': '37.59', 'tokens/total': 13710592, 'tokens/trainable': 207138, 'epoch': '1.77'} |
|
88%|████████▊ | 453/512 [50:23<06:38, 6.76s/it]
89%|████████▊ | 454/512 [50:30<06:29, 6.71s/it]
{'loss': '5.224e-05', 'grad_norm': '0.003731', 'learning_rate': '1.322e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.7', 'tokens/train_per_sec_per_gpu': '36.25', 'tokens/total': 13740960, 'tokens/trainable': 207583, 'epoch': '1.773'} |
|
89%|████████▊ | 454/512 [50:30<06:29, 6.71s/it]
89%|████████▉ | 455/512 [50:37<06:20, 6.68s/it]
{'loss': '0.0008256', 'grad_norm': '0.09384', 'learning_rate': '1.311e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '35.78', 'tokens/train_per_sec_per_gpu': '33.58', 'tokens/total': 13771296, 'tokens/trainable': 208025, 'epoch': '1.777'} |
|
89%|████████▉ | 455/512 [50:37<06:20, 6.68s/it]
89%|████████▉ | 456/512 [50:43<06:12, 6.66s/it]
{'loss': '0.001337', 'grad_norm': '0.1124', 'learning_rate': '1.301e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.86', 'memory/max_allocated (GiB)': '33.86', 'memory/device_reserved (GiB)': '35.78', 'tokens/train_per_sec_per_gpu': '35.32', 'tokens/total': 13801664, 'tokens/trainable': 208453, 'epoch': '1.781'} |
|
89%|████████▉ | 456/512 [50:43<06:12, 6.66s/it]
89%|████████▉ | 457/512 [50:50<06:05, 6.65s/it]
{'loss': '6.19e-05', 'grad_norm': '0.007983', 'learning_rate': '1.29e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '35.89', 'tokens/total': 13831936, 'tokens/trainable': 208947, 'epoch': '1.785'} |
|
89%|████████▉ | 457/512 [50:50<06:05, 6.65s/it]
89%|████████▉ | 458/512 [50:56<05:59, 6.66s/it]
{'loss': '9.091e-05', 'grad_norm': '0.00637', 'learning_rate': '1.28e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '35.33', 'tokens/total': 13862160, 'tokens/trainable': 209435, 'epoch': '1.789'} |
|
89%|████████▉ | 458/512 [50:56<05:59, 6.66s/it]
90%|████████▉ | 459/512 [51:03<05:52, 6.65s/it]
{'loss': '7.762e-05', 'grad_norm': '0.008154', 'learning_rate': '1.27e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.77', 'memory/max_allocated (GiB)': '33.77', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '33.98', 'tokens/total': 13892320, 'tokens/trainable': 209885, 'epoch': '1.793'} |
|
90%|████████▉ | 459/512 [51:03<05:52, 6.65s/it]
90%|████████▉ | 460/512 [51:10<05:45, 6.64s/it]
{'loss': '0.0001865', 'grad_norm': '0.01574', 'learning_rate': '1.26e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.91', 'memory/max_allocated (GiB)': '33.91', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '34.08', 'tokens/total': 13922752, 'tokens/trainable': 210344, 'epoch': '1.797'} |
|
90%|████████▉ | 460/512 [51:10<05:45, 6.64s/it]
90%|█████████ | 461/512 [51:16<05:37, 6.61s/it]
{'loss': '0.0001624', 'grad_norm': '0.01172', 'learning_rate': '1.251e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.65', 'memory/max_allocated (GiB)': '33.65', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '32.19', 'tokens/total': 13952816, 'tokens/trainable': 210814, 'epoch': '1.801'} |
|
90%|█████████ | 461/512 [51:16<05:37, 6.61s/it]
90%|█████████ | 462/512 [51:22<05:22, 6.46s/it]
{'loss': '0.0005623', 'grad_norm': '0.04298', 'learning_rate': '1.241e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.38', 'memory/max_allocated (GiB)': '33.38', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '30.87', 'tokens/total': 13981040, 'tokens/trainable': 211238, 'epoch': '1.805'} |
|
90%|█████████ | 462/512 [51:22<05:22, 6.46s/it]
90%|█████████ | 463/512 [51:29<05:18, 6.51s/it]
{'loss': '4.757e-05', 'grad_norm': '0.004571', 'learning_rate': '1.232e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.79', 'memory/max_allocated (GiB)': '33.79', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '35.64', 'tokens/total': 14011248, 'tokens/trainable': 211700, 'epoch': '1.809'} |
|
90%|█████████ | 463/512 [51:29<05:18, 6.51s/it]
91%|█████████ | 464/512 [51:36<05:13, 6.53s/it]
{'loss': '1.85e-05', 'grad_norm': '0.002208', 'learning_rate': '1.223e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.74', 'memory/max_allocated (GiB)': '33.74', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '29.47', 'tokens/total': 14041504, 'tokens/trainable': 212138, 'epoch': '1.812'} |
|
91%|█████████ | 464/512 [51:36<05:13, 6.53s/it]
91%|█████████ | 465/512 [51:42<05:08, 6.56s/it]
{'loss': '0.0008459', 'grad_norm': '0.08596', 'learning_rate': '1.214e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '33.14', 'tokens/total': 14071728, 'tokens/trainable': 212623, 'epoch': '1.816'} |
|
91%|█████████ | 465/512 [51:42<05:08, 6.56s/it]
91%|█████████ | 466/512 [51:49<05:02, 6.57s/it]
{'loss': '1.613e-05', 'grad_norm': '0.0007655', 'learning_rate': '1.205e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '33.37', 'tokens/total': 14101824, 'tokens/trainable': 213062, 'epoch': '1.82'} |
|
91%|█████████ | 466/512 [51:49<05:02, 6.57s/it]
91%|█████████ | 467/512 [51:55<04:56, 6.59s/it]
{'loss': '1.096e-05', 'grad_norm': '0.0005284', 'learning_rate': '1.197e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '32.03', 'tokens/total': 14132080, 'tokens/trainable': 213501, 'epoch': '1.824'} |
|
91%|█████████ | 467/512 [51:55<04:56, 6.59s/it]
91%|█████████▏| 468/512 [52:02<04:49, 6.58s/it]
{'loss': '1.107e-05', 'grad_norm': '0.0003768', 'learning_rate': '1.188e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.73', 'memory/max_allocated (GiB)': '33.73', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '35.13', 'tokens/total': 14162112, 'tokens/trainable': 213942, 'epoch': '1.828'} |
|
91%|█████████▏| 468/512 [52:02<04:49, 6.58s/it]
92%|█████████▏| 469/512 [52:09<04:43, 6.59s/it]
{'loss': '7.608e-05', 'grad_norm': '0.01082', 'learning_rate': '1.18e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '37.33', 'tokens/total': 14192784, 'tokens/trainable': 214410, 'epoch': '1.832'} |
|
92%|█████████▏| 469/512 [52:09<04:43, 6.59s/it]
92%|█████████▏| 470/512 [52:15<04:31, 6.46s/it]
{'loss': '0.0001094', 'grad_norm': '0.008473', 'learning_rate': '1.172e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '35.82', 'tokens/train_per_sec_per_gpu': '36.09', 'tokens/total': 14221008, 'tokens/trainable': 214844, 'epoch': '1.836'} |
|
92%|█████████▏| 470/512 [52:15<04:31, 6.46s/it]
92%|█████████▏| 471/512 [52:21<04:27, 6.53s/it]
{'loss': '1.788e-05', 'grad_norm': '0.0007592', 'learning_rate': '1.164e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '35.96', 'tokens/total': 14251536, 'tokens/trainable': 215336, 'epoch': '1.84'} |
|
92%|█████████▏| 471/512 [52:21<04:27, 6.53s/it]
92%|█████████▏| 472/512 [52:28<04:22, 6.55s/it]
{'loss': '0.0005082', 'grad_norm': '0.05926', 'learning_rate': '1.156e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '33.35', 'tokens/total': 14281664, 'tokens/trainable': 215777, 'epoch': '1.844'} |
|
92%|█████████▏| 472/512 [52:28<04:22, 6.55s/it]
92%|█████████▏| 473/512 [52:35<04:16, 6.58s/it]
{'loss': '0.002722', 'grad_norm': '0.1277', 'learning_rate': '1.149e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '35.25', 'tokens/total': 14312048, 'tokens/trainable': 216222, 'epoch': '1.848'} |
|
92%|█████████▏| 473/512 [52:35<04:16, 6.58s/it]
93%|█████████▎| 474/512 [52:41<04:09, 6.58s/it]
{'loss': '0.0001317', 'grad_norm': '0.02814', 'learning_rate': '1.142e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.76', 'memory/max_allocated (GiB)': '33.76', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '33.79', 'tokens/total': 14342112, 'tokens/trainable': 216662, 'epoch': '1.852'} |
|
93%|█████████▎| 474/512 [52:41<04:09, 6.58s/it]
93%|█████████▎| 475/512 [52:48<04:02, 6.56s/it]
{'loss': '5.48e-05', 'grad_norm': '0.003855', 'learning_rate': '1.135e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '31.26', 'tokens/total': 14372176, 'tokens/trainable': 217068, 'epoch': '1.855'} |
|
93%|█████████▎| 475/512 [52:48<04:02, 6.56s/it]
93%|█████████▎| 476/512 [52:54<03:57, 6.59s/it]
{'loss': '1.576e-05', 'grad_norm': '0.0007856', 'learning_rate': '1.128e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '38.21', 'tokens/total': 14402592, 'tokens/trainable': 217558, 'epoch': '1.859'} |
|
93%|█████████▎| 476/512 [52:54<03:57, 6.59s/it]
93%|█████████▎| 477/512 [53:01<03:50, 6.60s/it]
{'loss': '5.773e-05', 'grad_norm': '0.004396', 'learning_rate': '1.121e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.76', 'memory/max_allocated (GiB)': '33.76', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '35.98', 'tokens/total': 14432816, 'tokens/trainable': 218025, 'epoch': '1.863'} |
|
93%|█████████▎| 477/512 [53:01<03:50, 6.60s/it]
93%|█████████▎| 478/512 [53:08<03:46, 6.65s/it]
{'loss': '0.0003652', 'grad_norm': '0.03182', 'learning_rate': '1.114e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '32.81', 'tokens/total': 14463424, 'tokens/trainable': 218498, 'epoch': '1.867'} |
|
93%|█████████▎| 478/512 [53:08<03:46, 6.65s/it]
94%|█████████▎| 479/512 [53:14<03:38, 6.63s/it]
{'loss': '4.529e-05', 'grad_norm': '0.004733', 'learning_rate': '1.108e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.83', 'memory/max_allocated (GiB)': '33.83', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '33.17', 'tokens/total': 14493440, 'tokens/trainable': 218946, 'epoch': '1.871'} |
|
94%|█████████▎| 479/512 [53:14<03:38, 6.63s/it]
94%|█████████▍| 480/512 [53:21<03:32, 6.63s/it]
{'loss': '3.312e-05', 'grad_norm': '0.002677', 'learning_rate': '1.102e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.98', 'memory/max_allocated (GiB)': '33.98', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '34.52', 'tokens/total': 14524000, 'tokens/trainable': 219387, 'epoch': '1.875'} |
|
94%|█████████▍| 480/512 [53:21<03:32, 6.63s/it][2026-08-18 15:25:04,708] [INFO] [axolotl.core.trainers.base._save:828] [PID:12035] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-480 |
|
94%|█████████▍| 481/512 [53:30<03:44, 7.24s/it]
{'loss': '4.494e-05', 'grad_norm': '0.006715', 'learning_rate': '1.096e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.78', 'memory/max_allocated (GiB)': '33.78', 'memory/device_reserved (GiB)': '35.92', 'tokens/train_per_sec_per_gpu': '29.72', 'tokens/total': 14554016, 'tokens/trainable': 219813, 'epoch': '1.879'} |
|
94%|█████████▍| 481/512 [53:30<03:44, 7.24s/it]
94%|█████████▍| 482/512 [53:36<03:32, 7.08s/it]
{'loss': '0.0001471', 'grad_norm': '0.01935', 'learning_rate': '1.09e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '36.14', 'tokens/train_per_sec_per_gpu': '35.33', 'tokens/total': 14584512, 'tokens/trainable': 220321, 'epoch': '1.883'} |
|
94%|█████████▍| 482/512 [53:36<03:32, 7.08s/it]
94%|█████████▍| 483/512 [53:43<03:21, 6.94s/it]
{'loss': '3.409e-05', 'grad_norm': '0.005853', 'learning_rate': '1.084e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.8', 'memory/max_allocated (GiB)': '33.8', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '33.34', 'tokens/total': 14614608, 'tokens/trainable': 220751, 'epoch': '1.887'} |
|
94%|█████████▍| 483/512 [53:43<03:21, 6.94s/it]
95%|█████████▍| 484/512 [53:50<03:11, 6.85s/it]
{'loss': '1.306e-05', 'grad_norm': '0.0007543', 'learning_rate': '1.079e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.76', 'memory/max_allocated (GiB)': '33.76', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '35.32', 'tokens/total': 14644784, 'tokens/trainable': 221193, 'epoch': '1.891'} |
|
95%|█████████▍| 484/512 [53:50<03:11, 6.85s/it]
95%|█████████▍| 485/512 [53:56<03:02, 6.77s/it]
{'loss': '2.157e-05', 'grad_norm': '0.001243', 'learning_rate': '1.073e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.76', 'memory/max_allocated (GiB)': '33.76', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '32.04', 'tokens/total': 14674944, 'tokens/trainable': 221642, 'epoch': '1.895'} |
|
95%|█████████▍| 485/512 [53:56<03:02, 6.77s/it]
95%|█████████▍| 486/512 [54:03<02:54, 6.71s/it]
{'loss': '0.0001086', 'grad_norm': '0.01575', 'learning_rate': '1.068e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '31.93', 'tokens/total': 14705456, 'tokens/trainable': 222086, 'epoch': '1.898'} |
|
95%|█████████▍| 486/512 [54:03<02:54, 6.71s/it]
95%|█████████▌| 487/512 [54:09<02:46, 6.67s/it]
{'loss': '2.594e-05', 'grad_norm': '0.002205', 'learning_rate': '1.063e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.67', 'memory/max_allocated (GiB)': '33.67', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '32.44', 'tokens/total': 14735440, 'tokens/trainable': 222518, 'epoch': '1.902'} |
|
95%|█████████▌| 487/512 [54:09<02:46, 6.67s/it]
95%|█████████▌| 488/512 [54:16<02:39, 6.66s/it]
{'loss': '4.026e-05', 'grad_norm': '0.005776', 'learning_rate': '1.058e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '36.78', 'tokens/total': 14765856, 'tokens/trainable': 223011, 'epoch': '1.906'} |
|
95%|█████████▌| 488/512 [54:16<02:39, 6.66s/it]
96%|█████████▌| 489/512 [54:23<02:32, 6.63s/it]
{'loss': '6.143e-05', 'grad_norm': '0.008844', 'learning_rate': '1.054e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.77', 'memory/max_allocated (GiB)': '33.77', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '32.29', 'tokens/total': 14796128, 'tokens/trainable': 223464, 'epoch': '1.91'} |
|
96%|█████████▌| 489/512 [54:23<02:32, 6.63s/it]
96%|█████████▌| 490/512 [54:29<02:25, 6.61s/it]
{'loss': '9.643e-06', 'grad_norm': '0.0005093', 'learning_rate': '1.049e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.76', 'memory/max_allocated (GiB)': '33.76', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '29.84', 'tokens/total': 14826240, 'tokens/trainable': 223908, 'epoch': '1.914'} |
|
96%|█████████▌| 490/512 [54:29<02:25, 6.61s/it]
96%|█████████▌| 491/512 [54:36<02:18, 6.62s/it]
{'loss': '1.869e-05', 'grad_norm': '0.001238', 'learning_rate': '1.045e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '33.34', 'tokens/total': 14856512, 'tokens/trainable': 224365, 'epoch': '1.918'} |
|
96%|█████████▌| 491/512 [54:36<02:18, 6.62s/it]
96%|█████████▌| 492/512 [54:42<02:12, 6.63s/it]
{'loss': '0.0001639', 'grad_norm': '0.01618', 'learning_rate': '1.041e-05', 'ppl': '1', 'memory/max_active (GiB)': '34.01', 'memory/max_allocated (GiB)': '34.01', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '35.64', 'tokens/total': 14886848, 'tokens/trainable': 224847, 'epoch': '1.922'} |
|
96%|█████████▌| 492/512 [54:42<02:12, 6.63s/it]
96%|█████████▋| 493/512 [54:49<02:06, 6.64s/it]
{'loss': '0.01305', 'grad_norm': '0.573', 'learning_rate': '1.037e-05', 'ppl': '1.013', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '35.24', 'tokens/total': 14917216, 'tokens/trainable': 225329, 'epoch': '1.926'} |
|
96%|█████████▋| 493/512 [54:49<02:06, 6.64s/it]
96%|█████████▋| 494/512 [54:56<01:59, 6.64s/it]
{'loss': '1.876e-05', 'grad_norm': '0.00125', 'learning_rate': '1.034e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.84', 'memory/max_allocated (GiB)': '33.84', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '33.23', 'tokens/total': 14947648, 'tokens/trainable': 225778, 'epoch': '1.93'} |
|
96%|█████████▋| 494/512 [54:56<01:59, 6.64s/it]
97%|█████████▋| 495/512 [55:02<01:52, 6.62s/it]
{'loss': '7.826e-05', 'grad_norm': '0.009813', 'learning_rate': '1.03e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.81', 'memory/max_allocated (GiB)': '33.81', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '32.99', 'tokens/total': 14977728, 'tokens/trainable': 226217, 'epoch': '1.934'} |
|
97%|█████████▋| 495/512 [55:02<01:52, 6.62s/it]
97%|█████████▋| 496/512 [55:09<01:46, 6.64s/it]
{'loss': '3.328e-05', 'grad_norm': '0.003054', 'learning_rate': '1.027e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.9', 'memory/max_allocated (GiB)': '33.9', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '35.29', 'tokens/total': 15008032, 'tokens/trainable': 226689, 'epoch': '1.938'} |
|
97%|█████████▋| 496/512 [55:09<01:46, 6.64s/it]
97%|█████████▋| 497/512 [55:16<01:39, 6.62s/it]
{'loss': '0.0004042', 'grad_norm': '0.03506', 'learning_rate': '1.024e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '32.2', 'tokens/total': 15038528, 'tokens/trainable': 227112, 'epoch': '1.941'} |
|
97%|█████████▋| 497/512 [55:16<01:39, 6.62s/it]
97%|█████████▋| 498/512 [55:22<01:32, 6.61s/it]
{'loss': '2.265e-05', 'grad_norm': '0.001205', 'learning_rate': '1.021e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.87', 'memory/max_allocated (GiB)': '33.87', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '38.19', 'tokens/total': 15068832, 'tokens/trainable': 227577, 'epoch': '1.945'} |
|
97%|█████████▋| 498/512 [55:22<01:32, 6.61s/it]
97%|█████████▋| 499/512 [55:29<01:26, 6.62s/it]
{'loss': '5.452e-05', 'grad_norm': '0.002739', 'learning_rate': '1.018e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '37.11', 'tokens/total': 15099136, 'tokens/trainable': 228053, 'epoch': '1.949'} |
|
97%|█████████▋| 499/512 [55:29<01:26, 6.62s/it]
98%|█████████▊| 500/512 [55:35<01:19, 6.62s/it]
{'loss': '3.144e-05', 'grad_norm': '0.002191', 'learning_rate': '1.016e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.82', 'memory/max_allocated (GiB)': '33.82', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '31.33', 'tokens/total': 15129488, 'tokens/trainable': 228503, 'epoch': '1.953'} |
|
98%|█████████▊| 500/512 [55:35<01:19, 6.62s/it]
98%|█████████▊| 501/512 [55:42<01:12, 6.61s/it]
{'loss': '0.00122', 'grad_norm': '0.1152', 'learning_rate': '1.013e-05', 'ppl': '1.001', 'memory/max_active (GiB)': '33.92', 'memory/max_allocated (GiB)': '33.92', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '35.73', 'tokens/total': 15159840, 'tokens/trainable': 228983, 'epoch': '1.957'} |
|
98%|█████████▊| 501/512 [55:42<01:12, 6.61s/it]
98%|█████████▊| 502/512 [55:49<01:06, 6.63s/it]
{'loss': '4.653e-05', 'grad_norm': '0.008556', 'learning_rate': '1.011e-05', 'ppl': '1', 'memory/max_active (GiB)': '34.03', 'memory/max_allocated (GiB)': '34.03', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '31.47', 'tokens/total': 15190624, 'tokens/trainable': 229415, 'epoch': '1.961'} |
|
98%|█████████▊| 502/512 [55:49<01:06, 6.63s/it]
98%|█████████▊| 503/512 [55:55<00:59, 6.62s/it]
{'loss': '1.879e-05', 'grad_norm': '0.0009092', 'learning_rate': '1.009e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '33.25', 'tokens/total': 15221104, 'tokens/trainable': 229862, 'epoch': '1.965'} |
|
98%|█████████▊| 503/512 [55:55<00:59, 6.62s/it]
98%|█████████▊| 504/512 [56:02<00:53, 6.63s/it]
{'loss': '1.41e-05', 'grad_norm': '0.0007018', 'learning_rate': '1.008e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.96', 'memory/max_allocated (GiB)': '33.96', 'memory/device_reserved (GiB)': '36.15', 'tokens/train_per_sec_per_gpu': '39.42', 'tokens/total': 15251520, 'tokens/trainable': 230366, 'epoch': '1.969'} |
|
98%|█████████▊| 504/512 [56:02<00:53, 6.63s/it]
99%|█████████▊| 505/512 [56:09<00:46, 6.63s/it]
{'loss': '5.276e-05', 'grad_norm': '0.004301', 'learning_rate': '1.006e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.97', 'memory/max_allocated (GiB)': '33.97', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '38.02', 'tokens/total': 15282080, 'tokens/trainable': 230862, 'epoch': '1.973'} |
|
99%|█████████▊| 505/512 [56:09<00:46, 6.63s/it]
99%|█████████▉| 506/512 [56:15<00:39, 6.63s/it]
{'loss': '2.397e-05', 'grad_norm': '0.001468', 'learning_rate': '1.005e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.85', 'memory/max_allocated (GiB)': '33.85', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '34.73', 'tokens/total': 15312304, 'tokens/trainable': 231312, 'epoch': '1.977'} |
|
99%|█████████▉| 506/512 [56:15<00:39, 6.63s/it]
99%|█████████▉| 507/512 [56:22<00:33, 6.65s/it]
{'loss': '0.002683', 'grad_norm': '0.2848', 'learning_rate': '1.003e-05', 'ppl': '1.003', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '36.68', 'tokens/total': 15342912, 'tokens/trainable': 231801, 'epoch': '1.98'} |
|
99%|█████████▉| 507/512 [56:22<00:33, 6.65s/it]
99%|█████████▉| 508/512 [56:28<00:25, 6.49s/it]
{'loss': '0.0001258', 'grad_norm': '0.01371', 'learning_rate': '1.002e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.33', 'memory/max_allocated (GiB)': '33.33', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '31.96', 'tokens/total': 15371152, 'tokens/trainable': 232218, 'epoch': '1.984'} |
|
99%|█████████▉| 508/512 [56:28<00:25, 6.49s/it]
99%|█████████▉| 509/512 [56:35<00:19, 6.60s/it]
{'loss': '0.008881', 'grad_norm': '0.4796', 'learning_rate': '1.001e-05', 'ppl': '1.009', 'memory/max_active (GiB)': '34.02', 'memory/max_allocated (GiB)': '34.02', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '37.67', 'tokens/total': 15401984, 'tokens/trainable': 232718, 'epoch': '1.988'} |
|
99%|█████████▉| 509/512 [56:35<00:19, 6.60s/it]
100%|█████████▉| 510/512 [56:42<00:13, 6.62s/it]
{'loss': '0.0001719', 'grad_norm': '0.02159', 'learning_rate': '1.001e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.94', 'memory/max_allocated (GiB)': '33.94', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '36.58', 'tokens/total': 15432352, 'tokens/trainable': 233207, 'epoch': '1.992'} |
|
100%|█████████▉| 510/512 [56:42<00:13, 6.62s/it]
100%|█████████▉| 511/512 [56:48<00:06, 6.61s/it]
{'loss': '1.659e-05', 'grad_norm': '0.001061', 'learning_rate': '1e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.88', 'memory/max_allocated (GiB)': '33.88', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '30.82', 'tokens/total': 15462720, 'tokens/trainable': 233631, 'epoch': '1.996'} |
|
100%|█████████▉| 511/512 [56:48<00:06, 6.61s/it]
100%|██████████| 512/512 [56:55<00:00, 6.65s/it]
{'loss': '5.155e-05', 'grad_norm': '0.003317', 'learning_rate': '1e-05', 'ppl': '1', 'memory/max_active (GiB)': '33.89', 'memory/max_allocated (GiB)': '33.89', 'memory/device_reserved (GiB)': '36.16', 'tokens/train_per_sec_per_gpu': '36.62', 'tokens/total': 15493248, 'tokens/trainable': 234100, 'epoch': '2'} |
|
100%|██████████| 512/512 [56:55<00:00, 6.65s/it][2026-08-18 15:28:38,565] [INFO] [axolotl.core.trainers.base._save:828] [PID:12035] Saving model checkpoint to /workspace/wave/training/checkpoints/checkpoint-512 |
|
{'train_runtime': '3417', 'train_samples_per_second': '4.795', 'train_steps_per_second': '0.15', 'train_loss': '0.01031', 'memory/max_active (GiB)': '24.25', 'memory/max_allocated (GiB)': '24.25', 'memory/device_reserved (GiB)': '36.16', 'epoch': '2', 'tokens/train_per_sec_per_gpu': '0'} |
|
100%|██████████| 512/512 [56:57<00:00, 6.65s/it]
100%|██████████| 512/512 [56:57<00:00, 6.67s/it] |
| [2026-08-18 15:28:40,227] [INFO] [axolotl.train.save_trained_model:267] [PID:12035] Training completed! Saving trained model to /workspace/wave/training/checkpoints. |
| [2026-08-18 15:28:40,552] [INFO] [axolotl.train.save_trained_model:388] [PID:12035] Model successfully saved to /workspace/wave/training/checkpoints |
|
|
