add remote code + model files

.hydra/config.yaml

@@ -0,0 +1,93 @@
+model:
+  _target_: forgetting_transformer.model.alibi.modeling_alibi.AlibiForCausalLM
+  config:
+    _target_: forgetting_transformer.model.alibi.configuration_alibi.AlibiConfig
+    vocab_size: ???
+    hidden_size: 256
+    hidden_ratio: 4
+    intermediate_size: null
+    num_hidden_layers: 2
+    num_heads: 4
+    num_kv_heads: null
+    hidden_act: swish
+    window_size: null
+    max_position_embeddings: null
+    initializer_range: 0.02
+    elementwise_affine: true
+    norm_eps: 1.0e-06
+    use_cache: true
+    pad_token_id: null
+    bos_token_id: null
+    eos_token_id: null
+    tie_word_embeddings: false
+    attention_bias: false
+    fuse_norm: true
+    fuse_cross_entropy: true
+    use_rope: false
+    use_alibi: true
+optimizer:
+  _target_: torch.optim.AdamW
+  lr: 0.001
+  betas:
+  - 0.9
+  - 0.95
+  weight_decay: 0.1
+schedule:
+  _target_: forgetting_transformer.schedule.warmup_cosine_decay_schedule
+  init_value: 0.0
+  peak_value: ${optimizer.lr}
+  warmup_steps: 20971520
+  decay_steps: ${train.max_tokens}
+  end_value: 0.0
+datamodule:
+  _target_: forgetting_transformer.datamodule.npy.NpyDataModule
+  data_path: ${data_dir}
+  rank: ???
+  world_size: ???
+  train_batch_len: 2048
+  train_batch_size: 1024
+  train_num_workers: 0
+  eval_tokens: 2147483648
+  eval_batch_len: 2048
+  eval_local_batch_size: 1
+  eval_num_workers: 0
+strategy:
+  _target_: lightning.fabric.strategies.FSDPStrategy
+  state_dict_type: full
+  sharding_strategy: FULL_SHARD
+  cpu_offload: false
+exp: alibi_2_4_256
+tag: alibi_2_4_256
+seed: 42
+hf_load_dir: null
+hf_save_dir: null
+hf_load_step: null
+output_dir: ./alibi_2_4_256/
+data_dir: data
+resume: false
+fork_dir: null
+fork_step: null
+log_interval: 20971520
+eval_interval: 41943040
+final_eval: true
+skip_eval: false
+checkpoint_interval: 209715200
+train_eval_interval: 104857600
+checkpoint_keep_interval: 209715200
+fabric:
+  devices: 1
+  precision: 16-mixed
+train:
+  max_tokens: 2097152000
+  grad_acc_tokens: 32768
+  max_grad_norm: 1.0
+  gradient_checkpointing: false
+  bias_weight_decay: false
+  normalization_weight_decay: false
+  conv_weight_decay: true
+eval:
+  min_val_length: 512
+wandb:
+  project: forgetting-transformer
+  mode: online
+  log_dir: ./output/wandb

.hydra/hydra.yaml

@@ -0,0 +1,140 @@
+hydra:
+  run:
+    dir: ${output_dir}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    root: null
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    root: null
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - +experiment/pile/alibi=alibi_2_4_256
+    - fabric.devices=1
+    - fabric.precision=16-mixed
+    - seed=42
+    - exp=alibi_2_4_256
+    - tag=alibi_2_4_256
+    - output_dir=./alibi_2_4_256/
+    - wandb.log_dir=./output/wandb
+    - wandb.mode=online
+    - resume=false
+  job:
+    name: train
+    chdir: null
+    override_dirname: +experiment/pile/alibi=alibi_2_4_256,exp=alibi_2_4_256,fabric.devices=1,fabric.precision=16-mixed,output_dir=./alibi_2_4_256/,resume=false,seed=42,tag=alibi_2_4_256,wandb.log_dir=./output/wandb,wandb.mode=online
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /workspace/forgetting-transformer
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /workspace/forgetting-transformer/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /workspace/forgetting-transformer/alibi_2_4_256
+    choices:
+      experiment/pile/alibi: alibi_2_4_256
+      strategy: fsdp
+      datamodule: npy
+      schedule: warmup_cosine
+      optimizer: adamw
+      model: alibi
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: none
+      hydra/hydra_logging: none
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

.hydra/overrides.yaml

@@ -0,0 +1,10 @@
+- +experiment/pile/alibi=alibi_2_4_256
+- fabric.devices=1
+- fabric.precision=16-mixed
+- seed=42
+- exp=alibi_2_4_256
+- tag=alibi_2_4_256
+- output_dir=./alibi_2_4_256/
+- wandb.log_dir=./output/wandb
+- wandb.mode=online
+- resume=false

__init__.py

@@ -0,0 +1 @@
+# for HF remote code

checkpoints/step-000000209715200.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec850b83cd80f554cbaee361b572f4e98f46b3f38043b41115d6254c755d89b2
+size 329410370

checkpoints/step-000000419430400.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36d99734f17e6bd7ed7f3c6507fd9f42049ebcc506c8d2295132be0d28691596
+size 329410370

checkpoints/step-000000629145600.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:103930d0d1c0a46cfe767d05427b41c0dcda68c38c677313985b658a6072b0d3
+size 329410370

checkpoints/step-000000838860800.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87a4d22266e6afe2151eb7f240291f97b54289f17cef162b1878fc706eb59f8b
+size 329410370

checkpoints/step-000001048576000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b9206d508292b688c2a10f2c37f8c7e843c2b0a3845f6ec9fe9add1e773423c
+size 329410370

checkpoints/step-000001258291200.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:343fd80723001feb28b4fd1886c550885b02b39b3cb20dce26382e32835343f5
+size 329410370

checkpoints/step-000001468006400.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:097bf7b63cf0e604e2c2567d1cef7e399f58b74336a8b9f73b61d81f68d546e4
+size 329410370

checkpoints/step-000001677721600.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c1358a05b23cc9074bf3c29c10a719418029e63ca84a1661c7121aaa03e777e
+size 329410370

checkpoints/step-000001887436800.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cff1b93abb6e02112836ac20d76032e9051fd95369638b35ee3aa0cf5bf8553f
+size 329410370

config.yaml

@@ -0,0 +1,93 @@
+model:
+  _target_: forgetting_transformer.model.alibi.modeling_alibi.AlibiForCausalLM
+  config:
+    _target_: forgetting_transformer.model.alibi.configuration_alibi.AlibiConfig
+    vocab_size: ???
+    hidden_size: 256
+    hidden_ratio: 4
+    intermediate_size: null
+    num_hidden_layers: 2
+    num_heads: 4
+    num_kv_heads: null
+    hidden_act: swish
+    window_size: null
+    max_position_embeddings: null
+    initializer_range: 0.02
+    elementwise_affine: true
+    norm_eps: 1.0e-06
+    use_cache: true
+    pad_token_id: null
+    bos_token_id: null
+    eos_token_id: null
+    tie_word_embeddings: false
+    attention_bias: false
+    fuse_norm: true
+    fuse_cross_entropy: true
+    use_rope: false
+    use_alibi: true
+optimizer:
+  _target_: torch.optim.AdamW
+  lr: 0.001
+  betas:
+  - 0.9
+  - 0.95
+  weight_decay: 0.1
+schedule:
+  _target_: forgetting_transformer.schedule.warmup_cosine_decay_schedule
+  init_value: 0.0
+  peak_value: 0.001
+  warmup_steps: 20971520
+  decay_steps: 2097152000
+  end_value: 0.0
+datamodule:
+  _target_: forgetting_transformer.datamodule.npy.NpyDataModule
+  data_path: /workspace/forgetting-transformer/data
+  rank: ???
+  world_size: ???
+  train_batch_len: 2048
+  train_batch_size: 1024
+  train_num_workers: 0
+  eval_tokens: 2147483648
+  eval_batch_len: 2048
+  eval_local_batch_size: 1
+  eval_num_workers: 0
+strategy:
+  _target_: lightning.fabric.strategies.FSDPStrategy
+  state_dict_type: full
+  sharding_strategy: FULL_SHARD
+  cpu_offload: false
+exp: alibi_2_4_256
+tag: alibi_2_4_256
+seed: 42
+hf_load_dir: null
+hf_save_dir: null
+hf_load_step: null
+output_dir: /workspace/forgetting-transformer/alibi_2_4_256
+data_dir: /workspace/forgetting-transformer/data
+resume: false
+fork_dir: null
+fork_step: null
+log_interval: 20971520
+eval_interval: 41943040
+final_eval: true
+skip_eval: false
+checkpoint_interval: 209715200
+train_eval_interval: 104857600
+checkpoint_keep_interval: 209715200
+fabric:
+  devices: 1
+  precision: 16-mixed
+train:
+  max_tokens: 2097152000
+  grad_acc_tokens: 32768
+  max_grad_norm: 1.0
+  gradient_checkpointing: false
+  bias_weight_decay: false
+  normalization_weight_decay: false
+  conv_weight_decay: true
+eval:
+  min_val_length: 512
+wandb:
+  project: forgetting-transformer
+  mode: online
+  log_dir: ./output/wandb

configuration_transformer.py

@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+
+from typing import Optional
+
+from transformers.configuration_utils import PretrainedConfig
+
+
+class TransformerConfig(PretrainedConfig):
+
+    model_type = 'transformer-project_fox'
+    keys_to_ignore_at_inference = ['past_key_values']
+
+    def __init__(
+        self,
+        vocab_size: int = 32000,
+        hidden_size: int = 2048,
+        hidden_ratio: Optional[int] = 4,
+        intermediate_size: Optional[int] = None,
+        num_hidden_layers: int = 24,
+        num_heads: int = 32,
+        num_kv_heads: int = None,
+        hidden_act: str = "swish",
+        window_size: Optional[int] = None,
+        max_position_embeddings: int = 2048,
+        initializer_range: float = 0.02,
+        elementwise_affine: Optional[bool] = True,
+        norm_eps: float = 1e-6,
+        use_cache: bool = True,
+        pad_token_id: int = None,
+        bos_token_id: int = 1,
+        eos_token_id: int = 2,
+        tie_word_embeddings: bool = False,
+        attention_bias: bool = False,
+        fuse_norm: bool = True,
+        fuse_cross_entropy: bool = True,
+        rope_base: float = 500000.0,
+        use_rope: bool = True,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.hidden_ratio = hidden_ratio
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_heads = num_heads
+        self.num_kv_heads = num_kv_heads
+        self.window_size = window_size
+        self.max_position_embeddings = max_position_embeddings
+
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.elementwise_affine = elementwise_affine
+        self.norm_eps = norm_eps
+        self.use_cache = use_cache
+        self.attention_bias = attention_bias
+        self.fuse_cross_entropy = fuse_cross_entropy
+        self.fuse_norm = fuse_norm
+        self.rope_base = rope_base
+        self.use_rope = use_rope
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )

decay_params.txt

@@ -0,0 +1,14 @@
+_forward_module._fsdp_wrapped_module.model.embeddings.weight
+_forward_module._fsdp_wrapped_module.model.layers.0.attn.q_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.0.attn.k_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.0.attn.v_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.0.attn.o_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.0.mlp.gate_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.0.mlp.down_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.1.attn.q_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.1.attn.k_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.1.attn.v_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.1.attn.o_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.1.mlp.gate_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.1.mlp.down_proj.weight
+_forward_module._fsdp_wrapped_module.lm_head.weight

logs/2025-10-28_17-33-58.log

@@ -0,0 +1,258 @@
+[2025-10-28 17:33:59][train:372][INFO] All outputs will be saved to `/workspace/forgetting-transformer/alibi_2_4_256`
+[2025-10-28 17:33:59][train:375][INFO] Configuration:
+[2025-10-28 17:33:59][train:380][INFO] Configuration saved to /workspace/forgetting-transformer/alibi_2_4_256/config.yaml.
+[2025-10-28 17:33:59][train:387][INFO] creating datamodule
+[2025-10-28 17:33:59][train:419][INFO] creating model
+[2025-10-28 17:33:59][train:440][INFO] creating optimizer
+[2025-10-28 17:33:59][checkpoint:39][INFO] Not resuming. Deleting existing checkpoints...
+[2025-10-28 17:33:59][logger:256][INFO] Setting up wandb logger...
+[2025-10-28 17:33:59][logger:272][INFO] Not resuming. Creating a new wandb run.
+[2025-10-28 17:34:00][logger:288][INFO] wandb initialized. Run id: lo4di2up
+[2025-10-28 17:34:00][logger:186][INFO] Setting up jsonlines logger...
+[2025-10-28 17:34:00][logger:113][INFO] Setting up npz logger...
+[2025-10-28 17:34:00][logger:171][INFO] [step: 0] [train_data_info/vocab_size: 50277] [train_data_info/global_tokens_per_batch: 2097152] [train_data_info/local_tokens_per_batch: 2097152] [train_data_info/batch_len: 2048] [train_data_info/seq_len: 2048] [train_data_info/total_tokens: 2055208960] [train_data_info/global_batch_size: 1024] [train_data_info/local_batch_size: 1024]
+[2025-10-28 17:34:00][logger:171][INFO] [step: 0] [val_data_info/vocab_size: 50277] [val_data_info/global_tokens_per_batch: 2048] [val_data_info/local_tokens_per_batch: 2048] [val_data_info/batch_len: 2048] [val_data_info/seq_len: 2048] [val_data_info/total_tokens: 2147483648] [val_data_info/global_batch_size: 1] [val_data_info/local_batch_size: 1]
+[2025-10-28 17:34:00][logger:171][INFO] [step: 0] [model_info/total_params: 27447040] [model_info/trainable_params: 27447040] [model_info/embedding_params: 12870912] [model_info/flops_per_token: 0] [model_info/non_embedding_params: 14576128]
+[2025-10-28 17:35:00][utils:57][INFO] [P: 1.00%] [S: 20971520/2097152000] [T: 0:00:59] [ETA: 1:38:12] [loss: 9.762] [tokens/s: 374995.302] [batches/s: 0.179] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 17:35:56][utils:57][INFO] [P: 2.00%] [S: 41943040/2097152000] [T: 0:01:55] [ETA: 1:34:14] [loss: 8.127] [tokens/s: 375180.565] [batches/s: 0.179] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 17:35:56][train:194][INFO] Running validation...
+[2025-10-28 17:37:27][logger:171][INFO] [step: 41943040] [val/train_token_count: 41943040] [val/train_batch_count: 20] [val/train_flop_count: 0] [val/train_total_time: 115.389] [val/train_update_time: 115.085] [val/loss: 8.017] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.896] [val/val_tokens_per_second: 450625.947] [val/loss_avg_len_2048: 8.017] [val/perplexity_len_2048: 3033.046] [val/loss_avg_len_1024: 8.016] [val/perplexity_len_1024: 3029.389] [val/loss_avg_len_512: 8.017] [val/perplexity_len_512: 3030.799]
+[2025-10-28 17:38:23][utils:57][INFO] [P: 3.00%] [S: 62914560/2097152000] [T: 0:04:22] [ETA: 2:21:16] [loss: 7.520] [tokens/s: 240408.593] [batches/s: 0.115] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 17:39:19][utils:57][INFO] [P: 4.00%] [S: 83886080/2097152000] [T: 0:05:18] [ETA: 2:07:12] [loss: 7.193] [tokens/s: 264837.441] [batches/s: 0.126] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 17:39:19][train:194][INFO] Running validation...
+[2025-10-28 17:40:50][logger:171][INFO] [step: 83886080] [val/train_token_count: 83886080] [val/train_batch_count: 40] [val/train_flop_count: 0] [val/train_total_time: 318.011] [val/train_update_time: 226.570] [val/loss: 7.169] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 91.534] [val/val_tokens_per_second: 447486.078] [val/loss_avg_len_2048: 7.169] [val/perplexity_len_2048: 1298.286] [val/loss_avg_len_1024: 7.169] [val/perplexity_len_1024: 1298.841] [val/loss_avg_len_512: 7.173] [val/perplexity_len_512: 1303.146]
+[2025-10-28 17:41:46][utils:57][INFO] [P: 5.00%] [S: 104857600/2097152000] [T: 0:07:45] [ETA: 2:27:22] [loss: 6.947] [tokens/s: 225245.860] [batches/s: 0.107] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 17:41:46][logger:171][INFO] [step: 104857600] [train_eval/train_token_count: 104857600] [train_eval/train_batch_count: 50] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 465.399] [train_eval/train_update_time: 282.295] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 8.263] [train_eval/perplexity_len_2048: 3876.694] [train_eval/loss_avg_len_1024: 8.264] [train_eval/perplexity_len_1024: 3879.983] [train_eval/loss_avg_len_512: 8.264] [train_eval/perplexity_len_512: 3883.116]
+[2025-10-28 17:42:42][utils:57][INFO] [P: 6.00%] [S: 125829120/2097152000] [T: 0:08:41] [ETA: 2:16:06] [loss: 6.683] [tokens/s: 241633.348] [batches/s: 0.115] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 17:42:42][train:194][INFO] Running validation...
+[2025-10-28 17:44:13][logger:171][INFO] [step: 125829120] [val/train_token_count: 125829120] [val/train_batch_count: 60] [val/train_flop_count: 0] [val/train_total_time: 521.249] [val/train_update_time: 338.027] [val/loss: 6.682] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.733] [val/val_tokens_per_second: 451436.485] [val/loss_avg_len_2048: 6.682] [val/perplexity_len_2048: 797.878] [val/loss_avg_len_1024: 6.683] [val/perplexity_len_1024: 799.027] [val/loss_avg_len_512: 6.689] [val/perplexity_len_512: 803.835]
+[2025-10-28 17:45:08][utils:57][INFO] [P: 7.00%] [S: 146800640/2097152000] [T: 0:11:07] [ETA: 2:27:52] [loss: 6.482] [tokens/s: 219698.786] [batches/s: 0.105] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 17:46:04][utils:57][INFO] [P: 8.00%] [S: 167772160/2097152000] [T: 0:12:03] [ETA: 2:18:42] [loss: 6.282] [tokens/s: 231876.710] [batches/s: 0.111] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 17:46:04][train:194][INFO] Running validation...
+[2025-10-28 17:47:36][logger:171][INFO] [step: 167772160] [val/train_token_count: 167772160] [val/train_batch_count: 80] [val/train_flop_count: 0] [val/train_total_time: 723.681] [val/train_update_time: 449.489] [val/loss: 6.253] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 91.526] [val/val_tokens_per_second: 447522.667] [val/loss_avg_len_2048: 6.253] [val/perplexity_len_2048: 519.566] [val/loss_avg_len_1024: 6.256] [val/perplexity_len_1024: 521.125] [val/loss_avg_len_512: 6.265] [val/perplexity_len_512: 525.826]
+[2025-10-28 17:48:32][utils:57][INFO] [P: 9.00%] [S: 188743680/2097152000] [T: 0:14:31] [ETA: 2:26:47] [loss: 6.091] [tokens/s: 216563.422] [batches/s: 0.103] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 17:49:28][utils:57][INFO] [P: 10.00%] [S: 209715200/2097152000] [T: 0:15:26] [ETA: 2:19:01] [loss: 5.968] [tokens/s: 226236.908] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 17:49:28][logger:171][INFO] [step: 209715200] [train_eval/train_token_count: 209715200] [train_eval/train_batch_count: 100] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 926.886] [train_eval/train_update_time: 560.935] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 6.398] [train_eval/perplexity_len_2048: 600.871] [train_eval/loss_avg_len_1024: 6.403] [train_eval/perplexity_len_1024: 603.442] [train_eval/loss_avg_len_512: 6.409] [train_eval/perplexity_len_512: 607.288]
+[2025-10-28 17:49:28][train:194][INFO] Running validation...
+[2025-10-28 17:50:58][logger:171][INFO] [step: 209715200] [val/train_token_count: 209715200] [val/train_batch_count: 100] [val/train_flop_count: 0] [val/train_total_time: 926.886] [val/train_update_time: 560.935] [val/loss: 5.955] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.689] [val/val_tokens_per_second: 451651.287] [val/loss_avg_len_2048: 5.955] [val/perplexity_len_2048: 385.761] [val/loss_avg_len_1024: 5.959] [val/perplexity_len_1024: 387.331] [val/loss_avg_len_512: 5.970] [val/perplexity_len_512: 391.625]
+[2025-10-28 17:50:58][checkpoint:111][INFO] Saving checkpoint to /workspace/forgetting-transformer/alibi_2_4_256/checkpoints/step-000000209715200.pt...
+[2025-10-28 17:50:59][checkpoint:128][INFO] Checkpoint saved to /workspace/forgetting-transformer/alibi_2_4_256/checkpoints/step-000000209715200.pt.
+[2025-10-28 17:50:59][logger:171][INFO] [step: 209715200] [checkpoint/checkpoint_time: 0.452]
+[2025-10-28 17:51:55][utils:57][INFO] [P: 11.00%] [S: 230686720/2097152000] [T: 0:17:53] [ETA: 2:24:48] [loss: 5.855] [tokens/s: 205815.587] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 17:52:50][utils:57][INFO] [P: 12.00%] [S: 251658240/2097152000] [T: 0:18:49] [ETA: 2:18:04] [loss: 5.713] [tokens/s: 226206.211] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 17:52:50][train:194][INFO] Running validation...
+[2025-10-28 17:54:21][logger:171][INFO] [step: 251658240] [val/train_token_count: 251658240] [val/train_batch_count: 120] [val/train_flop_count: 0] [val/train_total_time: 1129.715] [val/train_update_time: 672.390] [val/loss: 5.728] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.612] [val/val_tokens_per_second: 452039.342] [val/loss_avg_len_2048: 5.728] [val/perplexity_len_2048: 307.227] [val/loss_avg_len_1024: 5.733] [val/perplexity_len_1024: 308.796] [val/loss_avg_len_512: 5.745] [val/perplexity_len_512: 312.716]
+[2025-10-28 17:55:17][utils:57][INFO] [P: 13.00%] [S: 272629760/2097152000] [T: 0:21:16] [ETA: 2:22:20] [loss: 5.645] [tokens/s: 205884.255] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 17:56:13][utils:57][INFO] [P: 14.00%] [S: 293601280/2097152000] [T: 0:22:12] [ETA: 2:16:22] [loss: 5.569] [tokens/s: 226441.230] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 17:56:13][train:194][INFO] Running validation...
+[2025-10-28 17:57:44][logger:171][INFO] [step: 293601280] [val/train_token_count: 293601280] [val/train_batch_count: 140] [val/train_flop_count: 0] [val/train_total_time: 1332.018] [val/train_update_time: 783.855] [val/loss: 5.546] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.990] [val/val_tokens_per_second: 450157.629] [val/loss_avg_len_2048: 5.546] [val/perplexity_len_2048: 256.270] [val/loss_avg_len_1024: 5.552] [val/perplexity_len_1024: 257.854] [val/loss_avg_len_512: 5.567] [val/perplexity_len_512: 261.533]
+[2025-10-28 17:58:40][utils:57][INFO] [P: 15.00%] [S: 314572800/2097152000] [T: 0:24:38] [ETA: 2:19:40] [loss: 5.447] [tokens/s: 205994.060] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 17:58:40][logger:171][INFO] [step: 314572800] [train_eval/train_token_count: 314572800] [train_eval/train_batch_count: 150] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 1478.875] [train_eval/train_update_time: 839.598] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 5.694] [train_eval/perplexity_len_2048: 297.217] [train_eval/loss_avg_len_1024: 5.700] [train_eval/perplexity_len_1024: 298.938] [train_eval/loss_avg_len_512: 5.712] [train_eval/perplexity_len_512: 302.436]
+[2025-10-28 17:59:35][utils:57][INFO] [P: 16.00%] [S: 335544320/2097152000] [T: 0:25:34] [ETA: 2:14:17] [loss: 5.413] [tokens/s: 226374.776] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 17:59:35][train:194][INFO] Running validation...
+[2025-10-28 18:01:06][logger:171][INFO] [step: 335544320] [val/train_token_count: 335544320] [val/train_batch_count: 160] [val/train_flop_count: 0] [val/train_total_time: 1534.725] [val/train_update_time: 895.339] [val/loss: 5.398] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.537] [val/val_tokens_per_second: 452412.827] [val/loss_avg_len_2048: 5.398] [val/perplexity_len_2048: 220.867] [val/loss_avg_len_1024: 5.405] [val/perplexity_len_1024: 222.462] [val/loss_avg_len_512: 5.420] [val/perplexity_len_512: 225.896]
+[2025-10-28 18:02:02][utils:57][INFO] [P: 17.00%] [S: 356515840/2097152000] [T: 0:28:01] [ETA: 2:16:47] [loss: 5.301] [tokens/s: 206033.343] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:02:58][utils:57][INFO] [P: 18.00%] [S: 377487360/2097152000] [T: 0:28:56] [ETA: 2:11:52] [loss: 5.273] [tokens/s: 226617.464] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:02:58][train:194][INFO] Running validation...
+[2025-10-28 18:04:28][logger:171][INFO] [step: 377487360] [val/train_token_count: 377487360] [val/train_batch_count: 180] [val/train_flop_count: 0] [val/train_total_time: 1736.966] [val/train_update_time: 1006.819] [val/loss: 5.267] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.563] [val/val_tokens_per_second: 452280.282] [val/loss_avg_len_2048: 5.267] [val/perplexity_len_2048: 193.790] [val/loss_avg_len_1024: 5.275] [val/perplexity_len_1024: 195.341] [val/loss_avg_len_512: 5.291] [val/perplexity_len_512: 198.610]
+[2025-10-28 18:05:24][utils:57][INFO] [P: 19.00%] [S: 398458880/2097152000] [T: 0:31:23] [ETA: 2:13:49] [loss: 5.245] [tokens/s: 206226.294] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:06:20][utils:57][INFO] [P: 20.00%] [S: 419430400/2097152000] [T: 0:32:19] [ETA: 2:09:16] [loss: 5.151] [tokens/s: 226752.548] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:06:20][logger:171][INFO] [step: 419430400] [train_eval/train_token_count: 419430400] [train_eval/train_batch_count: 200] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 1939.246] [train_eval/train_update_time: 1118.293] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 5.299] [train_eval/perplexity_len_2048: 200.122] [train_eval/loss_avg_len_1024: 5.307] [train_eval/perplexity_len_1024: 201.800] [train_eval/loss_avg_len_512: 5.322] [train_eval/perplexity_len_512: 204.871]
+[2025-10-28 18:06:20][train:194][INFO] Running validation...
+[2025-10-28 18:07:50][logger:171][INFO] [step: 419430400] [val/train_token_count: 419430400] [val/train_batch_count: 200] [val/train_flop_count: 0] [val/train_total_time: 1939.246] [val/train_update_time: 1118.293] [val/loss: 5.159] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.510] [val/val_tokens_per_second: 452546.668] [val/loss_avg_len_2048: 5.159] [val/perplexity_len_2048: 174.050] [val/loss_avg_len_1024: 5.168] [val/perplexity_len_1024: 175.577] [val/loss_avg_len_512: 5.186] [val/perplexity_len_512: 178.754]
+[2025-10-28 18:07:50][checkpoint:111][INFO] Saving checkpoint to /workspace/forgetting-transformer/alibi_2_4_256/checkpoints/step-000000419430400.pt...
+[2025-10-28 18:07:51][checkpoint:128][INFO] Checkpoint saved to /workspace/forgetting-transformer/alibi_2_4_256/checkpoints/step-000000419430400.pt.
+[2025-10-28 18:07:51][logger:171][INFO] [step: 419430400] [checkpoint/checkpoint_time: 0.443]
+[2025-10-28 18:08:47][utils:57][INFO] [P: 21.00%] [S: 440401920/2097152000] [T: 0:34:46] [ETA: 2:10:47] [loss: 5.107] [tokens/s: 206256.473] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:09:43][utils:57][INFO] [P: 22.00%] [S: 461373440/2097152000] [T: 0:35:41] [ETA: 2:06:34] [loss: 5.073] [tokens/s: 226657.953] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:09:43][train:194][INFO] Running validation...
+[2025-10-28 18:11:13][logger:171][INFO] [step: 461373440] [val/train_token_count: 461373440] [val/train_batch_count: 220] [val/train_flop_count: 0] [val/train_total_time: 2141.922] [val/train_update_time: 1229.772] [val/loss: 5.062] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.533] [val/val_tokens_per_second: 452433.127] [val/loss_avg_len_2048: 5.062] [val/perplexity_len_2048: 157.830] [val/loss_avg_len_1024: 5.071] [val/perplexity_len_1024: 159.347] [val/loss_avg_len_512: 5.090] [val/perplexity_len_512: 162.465]
+[2025-10-28 18:12:09][utils:57][INFO] [P: 23.00%] [S: 482344960/2097152000] [T: 0:38:08] [ETA: 2:07:40] [loss: 5.019] [tokens/s: 206268.387] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:13:05][utils:57][INFO] [P: 24.00%] [S: 503316480/2097152000] [T: 0:39:04] [ETA: 2:03:43] [loss: 4.965] [tokens/s: 226771.471] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:13:05][train:194][INFO] Running validation...
+[2025-10-28 18:14:36][logger:171][INFO] [step: 503316480] [val/train_token_count: 503316480] [val/train_batch_count: 240] [val/train_flop_count: 0] [val/train_total_time: 2344.148] [val/train_update_time: 1341.235] [val/loss: 4.984] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 91.017] [val/val_tokens_per_second: 450028.173] [val/loss_avg_len_2048: 4.984] [val/perplexity_len_2048: 146.091] [val/loss_avg_len_1024: 4.995] [val/perplexity_len_1024: 147.602] [val/loss_avg_len_512: 5.015] [val/perplexity_len_512: 150.648]
+[2025-10-28 18:15:32][utils:57][INFO] [P: 25.00%] [S: 524288000/2097152000] [T: 0:41:31] [ETA: 2:04:33] [loss: 4.966] [tokens/s: 206267.420] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:15:32][logger:171][INFO] [step: 524288000] [train_eval/train_token_count: 524288000] [train_eval/train_batch_count: 250] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 2491.022] [train_eval/train_update_time: 1396.971] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 5.048] [train_eval/perplexity_len_2048: 155.777] [train_eval/loss_avg_len_1024: 5.057] [train_eval/perplexity_len_1024: 157.044] [train_eval/loss_avg_len_512: 5.074] [train_eval/perplexity_len_512: 159.766]
+[2025-10-28 18:16:28][utils:57][INFO] [P: 26.00%] [S: 545259520/2097152000] [T: 0:42:26] [ETA: 2:00:48] [loss: 4.914] [tokens/s: 226652.272] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:16:28][train:194][INFO] Running validation...
+[2025-10-28 18:17:58][logger:171][INFO] [step: 545259520] [val/train_token_count: 545259520] [val/train_batch_count: 260] [val/train_flop_count: 0] [val/train_total_time: 2546.884] [val/train_update_time: 1452.724] [val/loss: 4.917] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.512] [val/val_tokens_per_second: 452536.960] [val/loss_avg_len_2048: 4.917] [val/perplexity_len_2048: 136.564] [val/loss_avg_len_1024: 4.927] [val/perplexity_len_1024: 138.032] [val/loss_avg_len_512: 4.949] [val/perplexity_len_512: 140.976]
+[2025-10-28 18:18:54][utils:57][INFO] [P: 27.00%] [S: 566231040/2097152000] [T: 0:44:53] [ETA: 2:01:21] [loss: 4.898] [tokens/s: 206268.199] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:19:50][utils:57][INFO] [P: 28.00%] [S: 587202560/2097152000] [T: 0:45:49] [ETA: 1:57:49] [loss: 4.850] [tokens/s: 226664.200] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:19:50][train:194][INFO] Running validation...
+[2025-10-28 18:21:20][logger:171][INFO] [step: 587202560] [val/train_token_count: 587202560] [val/train_batch_count: 280] [val/train_flop_count: 0] [val/train_total_time: 2749.101] [val/train_update_time: 1564.203] [val/loss: 4.861] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.514] [val/val_tokens_per_second: 452527.805] [val/loss_avg_len_2048: 4.861] [val/perplexity_len_2048: 129.097] [val/loss_avg_len_1024: 4.872] [val/perplexity_len_1024: 130.570] [val/loss_avg_len_512: 4.894] [val/perplexity_len_512: 133.488]
+[2025-10-28 18:22:16][utils:57][INFO] [P: 29.00%] [S: 608174080/2097152000] [T: 0:48:15] [ETA: 1:58:08] [loss: 4.818] [tokens/s: 206279.914] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:23:12][utils:57][INFO] [P: 30.00%] [S: 629145600/2097152000] [T: 0:49:11] [ETA: 1:54:46] [loss: 4.798] [tokens/s: 226778.201] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:23:12][logger:171][INFO] [step: 629145600] [train_eval/train_token_count: 629145600] [train_eval/train_batch_count: 300] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 2951.316] [train_eval/train_update_time: 1675.688] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.882] [train_eval/perplexity_len_2048: 131.873] [train_eval/loss_avg_len_1024: 4.890] [train_eval/perplexity_len_1024: 132.942] [train_eval/loss_avg_len_512: 4.909] [train_eval/perplexity_len_512: 135.482]
+[2025-10-28 18:23:12][train:194][INFO] Running validation...
+[2025-10-28 18:24:42][logger:171][INFO] [step: 629145600] [val/train_token_count: 629145600] [val/train_batch_count: 300] [val/train_flop_count: 0] [val/train_total_time: 2951.316] [val/train_update_time: 1675.688] [val/loss: 4.812] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.517] [val/val_tokens_per_second: 452511.126] [val/loss_avg_len_2048: 4.812] [val/perplexity_len_2048: 122.947] [val/loss_avg_len_1024: 4.824] [val/perplexity_len_1024: 124.414] [val/loss_avg_len_512: 4.847] [val/perplexity_len_512: 127.337]
+[2025-10-28 18:24:42][checkpoint:111][INFO] Saving checkpoint to /workspace/forgetting-transformer/alibi_2_4_256/checkpoints/step-000000629145600.pt...
+[2025-10-28 18:24:43][checkpoint:128][INFO] Checkpoint saved to /workspace/forgetting-transformer/alibi_2_4_256/checkpoints/step-000000629145600.pt.
+[2025-10-28 18:24:43][logger:171][INFO] [step: 629145600] [checkpoint/checkpoint_time: 0.436]
+[2025-10-28 18:25:39][utils:57][INFO] [P: 31.00%] [S: 650117120/2097152000] [T: 0:51:38] [ETA: 1:54:55] [loss: 4.810] [tokens/s: 206285.423] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:26:35][utils:57][INFO] [P: 32.00%] [S: 671088640/2097152000] [T: 0:52:33] [ETA: 1:51:42] [loss: 4.744] [tokens/s: 226677.414] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:26:35][train:194][INFO] Running validation...
+[2025-10-28 18:28:05][logger:171][INFO] [step: 671088640] [val/train_token_count: 671088640] [val/train_batch_count: 320] [val/train_flop_count: 0] [val/train_total_time: 3153.973] [val/train_update_time: 1787.177] [val/loss: 4.760] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.763] [val/val_tokens_per_second: 451284.325] [val/loss_avg_len_2048: 4.760] [val/perplexity_len_2048: 116.720] [val/loss_avg_len_1024: 4.772] [val/perplexity_len_1024: 118.146] [val/loss_avg_len_512: 4.796] [val/perplexity_len_512: 120.981]
+[2025-10-28 18:29:01][utils:57][INFO] [P: 33.00%] [S: 692060160/2097152000] [T: 0:55:00] [ETA: 1:51:41] [loss: 4.759] [tokens/s: 206236.434] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:29:57][utils:57][INFO] [P: 34.00%] [S: 713031680/2097152000] [T: 0:55:56] [ETA: 1:48:35] [loss: 4.724] [tokens/s: 226734.784] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:29:57][train:194][INFO] Running validation...
+[2025-10-28 18:31:28][logger:171][INFO] [step: 713031680] [val/train_token_count: 713031680] [val/train_batch_count: 340] [val/train_flop_count: 0] [val/train_total_time: 3356.451] [val/train_update_time: 1898.671] [val/loss: 4.717] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.574] [val/val_tokens_per_second: 452225.537] [val/loss_avg_len_2048: 4.717] [val/perplexity_len_2048: 111.799] [val/loss_avg_len_1024: 4.730] [val/perplexity_len_1024: 113.246] [val/loss_avg_len_512: 4.754] [val/perplexity_len_512: 116.073]
+[2025-10-28 18:32:24][utils:57][INFO] [P: 35.00%] [S: 734003200/2097152000] [T: 0:58:22] [ETA: 1:48:25] [loss: 4.704] [tokens/s: 206325.847] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:32:24][logger:171][INFO] [step: 734003200] [train_eval/train_token_count: 734003200] [train_eval/train_batch_count: 350] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 3502.878] [train_eval/train_update_time: 1954.413] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.752] [train_eval/perplexity_len_2048: 115.806] [train_eval/loss_avg_len_1024: 4.763] [train_eval/perplexity_len_1024: 117.122] [train_eval/loss_avg_len_512: 4.785] [train_eval/perplexity_len_512: 119.713]
+[2025-10-28 18:33:19][utils:57][INFO] [P: 36.00%] [S: 754974720/2097152000] [T: 0:59:18] [ETA: 1:45:26] [loss: 4.634] [tokens/s: 226723.796] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:33:19][train:194][INFO] Running validation...
+[2025-10-28 18:34:50][logger:171][INFO] [step: 754974720] [val/train_token_count: 754974720] [val/train_batch_count: 360] [val/train_flop_count: 0] [val/train_total_time: 3558.727] [val/train_update_time: 2010.155] [val/loss: 4.678] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.670] [val/val_tokens_per_second: 451747.243] [val/loss_avg_len_2048: 4.678] [val/perplexity_len_2048: 107.532] [val/loss_avg_len_1024: 4.691] [val/perplexity_len_1024: 108.985] [val/loss_avg_len_512: 4.717] [val/perplexity_len_512: 111.799]
+[2025-10-28 18:35:46][utils:57][INFO] [P: 37.00%] [S: 775946240/2097152000] [T: 1:01:45] [ETA: 1:45:08] [loss: 4.657] [tokens/s: 206294.493] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:36:42][utils:57][INFO] [P: 38.00%] [S: 796917760/2097152000] [T: 1:02:41] [ETA: 1:42:16] [loss: 4.635] [tokens/s: 226682.610] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:36:42][train:194][INFO] Running validation...
+[2025-10-28 18:38:12][logger:171][INFO] [step: 796917760] [val/train_token_count: 796917760] [val/train_batch_count: 380] [val/train_flop_count: 0] [val/train_total_time: 3761.110] [val/train_update_time: 2121.661] [val/loss: 4.641] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.679] [val/val_tokens_per_second: 451703.774] [val/loss_avg_len_2048: 4.641] [val/perplexity_len_2048: 103.622] [val/loss_avg_len_1024: 4.655] [val/perplexity_len_1024: 105.078] [val/loss_avg_len_512: 4.681] [val/perplexity_len_512: 107.891]
+[2025-10-28 18:39:08][utils:57][INFO] [P: 39.00%] [S: 817889280/2097152000] [T: 1:05:07] [ETA: 1:41:51] [loss: 4.639] [tokens/s: 206257.892] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:40:04][utils:57][INFO] [P: 40.00%] [S: 838860800/2097152000] [T: 1:06:03] [ETA: 1:39:05] [loss: 4.546] [tokens/s: 226752.243] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:40:04][logger:171][INFO] [step: 838860800] [train_eval/train_token_count: 838860800] [train_eval/train_batch_count: 400] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 3963.490] [train_eval/train_update_time: 2233.150] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.653] [train_eval/perplexity_len_2048: 104.895] [train_eval/loss_avg_len_1024: 4.664] [train_eval/perplexity_len_1024: 106.056] [train_eval/loss_avg_len_512: 4.688] [train_eval/perplexity_len_512: 108.609]
+[2025-10-28 18:40:04][train:194][INFO] Running validation...
+[2025-10-28 18:41:35][logger:171][INFO] [step: 838860800] [val/train_token_count: 838860800] [val/train_batch_count: 400] [val/train_flop_count: 0] [val/train_total_time: 3963.490] [val/train_update_time: 2233.150] [val/loss: 4.608] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.607] [val/val_tokens_per_second: 452063.792] [val/loss_avg_len_2048: 4.608] [val/perplexity_len_2048: 100.245] [val/loss_avg_len_1024: 4.622] [val/perplexity_len_1024: 101.732] [val/loss_avg_len_512: 4.650] [val/perplexity_len_512: 104.589]
+[2025-10-28 18:41:35][checkpoint:111][INFO] Saving checkpoint to /workspace/forgetting-transformer/alibi_2_4_256/checkpoints/step-000000838860800.pt...
+[2025-10-28 18:41:35][checkpoint:128][INFO] Checkpoint saved to /workspace/forgetting-transformer/alibi_2_4_256/checkpoints/step-000000838860800.pt.
+[2025-10-28 18:41:35][logger:171][INFO] [step: 838860800] [checkpoint/checkpoint_time: 0.443]
+[2025-10-28 18:42:31][utils:57][INFO] [P: 41.00%] [S: 859832320/2097152000] [T: 1:08:30] [ETA: 1:38:34] [loss: 4.568] [tokens/s: 206238.018] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:43:27][utils:57][INFO] [P: 42.00%] [S: 880803840/2097152000] [T: 1:09:26] [ETA: 1:35:53] [loss: 4.565] [tokens/s: 226680.204] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:43:27][train:194][INFO] Running validation...
+[2025-10-28 18:44:57][logger:171][INFO] [step: 880803840] [val/train_token_count: 880803840] [val/train_batch_count: 420] [val/train_flop_count: 0] [val/train_total_time: 4166.243] [val/train_update_time: 2344.623] [val/loss: 4.578] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.583] [val/val_tokens_per_second: 452180.051] [val/loss_avg_len_2048: 4.578] [val/perplexity_len_2048: 97.333] [val/loss_avg_len_1024: 4.594] [val/perplexity_len_1024: 98.844] [val/loss_avg_len_512: 4.622] [val/perplexity_len_512: 101.702]
+[2025-10-28 18:45:53][utils:57][INFO] [P: 43.00%] [S: 901775360/2097152000] [T: 1:11:52] [ETA: 1:35:16] [loss: 4.560] [tokens/s: 206280.440] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:46:49][utils:57][INFO] [P: 44.00%] [S: 922746880/2097152000] [T: 1:12:48] [ETA: 1:32:39] [loss: 4.582] [tokens/s: 226681.934] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:46:49][train:194][INFO] Running validation...
+[2025-10-28 18:48:20][logger:171][INFO] [step: 922746880] [val/train_token_count: 922746880] [val/train_batch_count: 440] [val/train_flop_count: 0] [val/train_total_time: 4368.526] [val/train_update_time: 2456.116] [val/loss: 4.545] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.743] [val/val_tokens_per_second: 451384.214] [val/loss_avg_len_2048: 4.545] [val/perplexity_len_2048: 94.177] [val/loss_avg_len_1024: 4.561] [val/perplexity_len_1024: 95.677] [val/loss_avg_len_512: 4.590] [val/perplexity_len_512: 98.539]
+[2025-10-28 18:49:16][utils:57][INFO] [P: 45.00%] [S: 943718400/2097152000] [T: 1:15:15] [ETA: 1:31:58] [loss: 4.528] [tokens/s: 206247.110] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:49:16][logger:171][INFO] [step: 943718400] [train_eval/train_token_count: 943718400] [train_eval/train_batch_count: 450] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 4515.121] [train_eval/train_update_time: 2511.850] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.568] [train_eval/perplexity_len_2048: 96.398] [train_eval/loss_avg_len_1024: 4.583] [train_eval/perplexity_len_1024: 97.828] [train_eval/loss_avg_len_512: 4.611] [train_eval/perplexity_len_512: 100.631]
+[2025-10-28 18:50:12][utils:57][INFO] [P: 46.00%] [S: 964689920/2097152000] [T: 1:16:10] [ETA: 1:29:25] [loss: 4.494] [tokens/s: 226662.302] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:50:12][train:194][INFO] Running validation...
+[2025-10-28 18:51:42][logger:171][INFO] [step: 964689920] [val/train_token_count: 964689920] [val/train_batch_count: 460] [val/train_flop_count: 0] [val/train_total_time: 4570.982] [val/train_update_time: 2567.596] [val/loss: 4.518] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.593] [val/val_tokens_per_second: 452130.146] [val/loss_avg_len_2048: 4.518] [val/perplexity_len_2048: 91.612] [val/loss_avg_len_1024: 4.534] [val/perplexity_len_1024: 93.126] [val/loss_avg_len_512: 4.564] [val/perplexity_len_512: 96.009]
+[2025-10-28 18:52:38][utils:57][INFO] [P: 47.00%] [S: 985661440/2097152000] [T: 1:18:37] [ETA: 1:28:39] [loss: 4.520] [tokens/s: 206261.701] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:53:34][utils:57][INFO] [P: 48.00%] [S: 1006632960/2097152000] [T: 1:19:33] [ETA: 1:26:11] [loss: 4.494] [tokens/s: 226681.695] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:53:34][train:194][INFO] Running validation...
+[2025-10-28 18:55:05][logger:171][INFO] [step: 1006632960] [val/train_token_count: 1006632960] [val/train_batch_count: 480] [val/train_flop_count: 0] [val/train_total_time: 4773.293] [val/train_update_time: 2679.073] [val/loss: 4.491] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.618] [val/val_tokens_per_second: 452007.561] [val/loss_avg_len_2048: 4.491] [val/perplexity_len_2048: 89.181] [val/loss_avg_len_1024: 4.508] [val/perplexity_len_1024: 90.714] [val/loss_avg_len_512: 4.539] [val/perplexity_len_512: 93.629]
+[2025-10-28 18:56:00][utils:57][INFO] [P: 49.00%] [S: 1027604480/2097152000] [T: 1:21:59] [ETA: 1:25:20] [loss: 4.490] [tokens/s: 206272.262] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:56:56][utils:57][INFO] [P: 50.00%] [S: 1048576000/2097152000] [T: 1:22:55] [ETA: 1:22:55] [loss: 4.463] [tokens/s: 226784.168] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 18:56:56][logger:171][INFO] [step: 1048576000] [train_eval/train_token_count: 1048576000] [train_eval/train_batch_count: 500] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 4975.631] [train_eval/train_update_time: 2790.558] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.499] [train_eval/perplexity_len_2048: 89.960] [train_eval/loss_avg_len_1024: 4.514] [train_eval/perplexity_len_1024: 91.275] [train_eval/loss_avg_len_512: 4.544] [train_eval/perplexity_len_512: 94.106]
+[2025-10-28 18:56:56][train:194][INFO] Running validation...
+[2025-10-28 18:58:27][logger:171][INFO] [step: 1048576000] [val/train_token_count: 1048576000] [val/train_batch_count: 500] [val/train_flop_count: 0] [val/train_total_time: 4975.631] [val/train_update_time: 2790.558] [val/loss: 4.466] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.630] [val/val_tokens_per_second: 451946.929] [val/loss_avg_len_2048: 4.466] [val/perplexity_len_2048: 87.051] [val/loss_avg_len_1024: 4.484] [val/perplexity_len_1024: 88.594] [val/loss_avg_len_512: 4.517] [val/perplexity_len_512: 91.528]
+[2025-10-28 18:58:27][checkpoint:111][INFO] Saving checkpoint to /workspace/forgetting-transformer/alibi_2_4_256/checkpoints/step-000001048576000.pt...
+[2025-10-28 18:58:27][checkpoint:128][INFO] Checkpoint saved to /workspace/forgetting-transformer/alibi_2_4_256/checkpoints/step-000001048576000.pt.
+[2025-10-28 18:58:27][logger:171][INFO] [step: 1048576000] [checkpoint/checkpoint_time: 0.438]
+[2025-10-28 18:59:23][utils:57][INFO] [P: 51.00%] [S: 1069547520/2097152000] [T: 1:25:22] [ETA: 1:22:01] [loss: 4.463] [tokens/s: 206262.376] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:00:19][utils:57][INFO] [P: 52.00%] [S: 1090519040/2097152000] [T: 1:26:18] [ETA: 1:19:40] [loss: 4.453] [tokens/s: 226659.051] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:00:19][train:194][INFO] Running validation...
+[2025-10-28 19:01:50][logger:171][INFO] [step: 1090519040] [val/train_token_count: 1090519040] [val/train_batch_count: 520] [val/train_flop_count: 0] [val/train_total_time: 5178.418] [val/train_update_time: 2902.025] [val/loss: 4.441] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.618] [val/val_tokens_per_second: 452008.856] [val/loss_avg_len_2048: 4.441] [val/perplexity_len_2048: 84.893] [val/loss_avg_len_1024: 4.460] [val/perplexity_len_1024: 86.474] [val/loss_avg_len_512: 4.494] [val/perplexity_len_512: 89.458]
+[2025-10-28 19:02:46][utils:57][INFO] [P: 53.00%] [S: 1111490560/2097152000] [T: 1:28:44] [ETA: 1:18:42] [loss: 4.414] [tokens/s: 206256.696] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:03:41][utils:57][INFO] [P: 54.00%] [S: 1132462080/2097152000] [T: 1:29:40] [ETA: 1:16:23] [loss: 4.424] [tokens/s: 226693.969] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:03:41][train:194][INFO] Running validation...
+[2025-10-28 19:05:12][logger:171][INFO] [step: 1132462080] [val/train_token_count: 1132462080] [val/train_batch_count: 540] [val/train_flop_count: 0] [val/train_total_time: 5380.720] [val/train_update_time: 3013.473] [val/loss: 4.421] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.536] [val/val_tokens_per_second: 452416.614] [val/loss_avg_len_2048: 4.421] [val/perplexity_len_2048: 83.142] [val/loss_avg_len_1024: 4.440] [val/perplexity_len_1024: 84.760] [val/loss_avg_len_512: 4.475] [val/perplexity_len_512: 87.782]
+[2025-10-28 19:06:08][utils:57][INFO] [P: 55.00%] [S: 1153433600/2097152000] [T: 1:32:07] [ETA: 1:15:22] [loss: 4.370] [tokens/s: 206300.113] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:06:08][logger:171][INFO] [step: 1153433600] [train_eval/train_token_count: 1153433600] [train_eval/train_batch_count: 550] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 5527.109] [train_eval/train_update_time: 3069.210] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.433] [train_eval/perplexity_len_2048: 84.219] [train_eval/loss_avg_len_1024: 4.448] [train_eval/perplexity_len_1024: 85.438] [train_eval/loss_avg_len_512: 4.479] [train_eval/perplexity_len_512: 88.131]
+[2025-10-28 19:07:04][utils:57][INFO] [P: 56.00%] [S: 1174405120/2097152000] [T: 1:33:02] [ETA: 1:13:06] [loss: 4.411] [tokens/s: 226710.951] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:07:04][train:194][INFO] Running validation...
+[2025-10-28 19:08:34][logger:171][INFO] [step: 1174405120] [val/train_token_count: 1174405120] [val/train_batch_count: 560] [val/train_flop_count: 0] [val/train_total_time: 5582.959] [val/train_update_time: 3124.938] [val/loss: 4.396] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.502] [val/val_tokens_per_second: 452587.756] [val/loss_avg_len_2048: 4.396] [val/perplexity_len_2048: 81.128] [val/loss_avg_len_1024: 4.416] [val/perplexity_len_1024: 82.762] [val/loss_avg_len_512: 4.452] [val/perplexity_len_512: 85.827]
+[2025-10-28 19:09:30][utils:57][INFO] [P: 57.00%] [S: 1195376640/2097152000] [T: 1:35:29] [ETA: 1:12:02] [loss: 4.361] [tokens/s: 206324.776] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:10:26][utils:57][INFO] [P: 58.00%] [S: 1216348160/2097152000] [T: 1:36:25] [ETA: 1:09:49] [loss: 4.409] [tokens/s: 226752.429] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:10:26][train:194][INFO] Running validation...
+[2025-10-28 19:11:56][logger:171][INFO] [step: 1216348160] [val/train_token_count: 1216348160] [val/train_batch_count: 580] [val/train_flop_count: 0] [val/train_total_time: 5785.136] [val/train_update_time: 3236.387] [val/loss: 4.375] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.566] [val/val_tokens_per_second: 452264.749] [val/loss_avg_len_2048: 4.375] [val/perplexity_len_2048: 79.412] [val/loss_avg_len_1024: 4.396] [val/perplexity_len_1024: 81.097] [val/loss_avg_len_512: 4.434] [val/perplexity_len_512: 84.267]
+[2025-10-28 19:12:52][utils:57][INFO] [P: 59.00%] [S: 1237319680/2097152000] [T: 1:38:51] [ETA: 1:08:41] [loss: 4.390] [tokens/s: 206343.676] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:13:48][utils:57][INFO] [P: 60.00%] [S: 1258291200/2097152000] [T: 1:39:47] [ETA: 1:06:31] [loss: 4.390] [tokens/s: 226890.200] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:13:48][logger:171][INFO] [step: 1258291200] [train_eval/train_token_count: 1258291200] [train_eval/train_batch_count: 600] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 5987.367] [train_eval/train_update_time: 3347.845] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.377] [train_eval/perplexity_len_2048: 79.612] [train_eval/loss_avg_len_1024: 4.393] [train_eval/perplexity_len_1024: 80.855] [train_eval/loss_avg_len_512: 4.428] [train_eval/perplexity_len_512: 83.760]
+[2025-10-28 19:13:48][train:194][INFO] Running validation...
+[2025-10-28 19:15:19][logger:171][INFO] [step: 1258291200] [val/train_token_count: 1258291200] [val/train_batch_count: 600] [val/train_flop_count: 0] [val/train_total_time: 5987.367] [val/train_update_time: 3347.845] [val/loss: 4.356] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.808] [val/val_tokens_per_second: 451063.348] [val/loss_avg_len_2048: 4.356] [val/perplexity_len_2048: 77.921] [val/loss_avg_len_1024: 4.378] [val/perplexity_len_1024: 79.645] [val/loss_avg_len_512: 4.417] [val/perplexity_len_512: 82.879]
+[2025-10-28 19:15:19][checkpoint:111][INFO] Saving checkpoint to /workspace/forgetting-transformer/alibi_2_4_256/checkpoints/step-000001258291200.pt...
+[2025-10-28 19:15:19][checkpoint:128][INFO] Checkpoint saved to /workspace/forgetting-transformer/alibi_2_4_256/checkpoints/step-000001258291200.pt.
+[2025-10-28 19:15:19][logger:171][INFO] [step: 1258291200] [checkpoint/checkpoint_time: 0.446]
+[2025-10-28 19:16:15][utils:57][INFO] [P: 61.00%] [S: 1279262720/2097152000] [T: 1:42:14] [ETA: 1:05:22] [loss: 4.355] [tokens/s: 206316.288] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:17:11][utils:57][INFO] [P: 62.00%] [S: 1300234240/2097152000] [T: 1:43:10] [ETA: 1:03:14] [loss: 4.328] [tokens/s: 226735.707] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:17:11][train:194][INFO] Running validation...
+[2025-10-28 19:18:41][logger:171][INFO] [step: 1300234240] [val/train_token_count: 1300234240] [val/train_batch_count: 620] [val/train_flop_count: 0] [val/train_total_time: 6190.320] [val/train_update_time: 3459.312] [val/loss: 4.334] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.454] [val/val_tokens_per_second: 452824.486] [val/loss_avg_len_2048: 4.334] [val/perplexity_len_2048: 76.238] [val/loss_avg_len_1024: 4.357] [val/perplexity_len_1024: 78.001] [val/loss_avg_len_512: 4.398] [val/perplexity_len_512: 81.297]
+[2025-10-28 19:19:37][utils:57][INFO] [P: 63.00%] [S: 1321205760/2097152000] [T: 1:45:36] [ETA: 1:02:01] [loss: 4.339] [tokens/s: 206342.439] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:20:33][utils:57][INFO] [P: 64.00%] [S: 1342177280/2097152000] [T: 1:46:32] [ETA: 0:59:55] [loss: 4.330] [tokens/s: 226742.795] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:20:33][train:194][INFO] Running validation...
+[2025-10-28 19:22:04][logger:171][INFO] [step: 1342177280] [val/train_token_count: 1342177280] [val/train_batch_count: 640] [val/train_flop_count: 0] [val/train_total_time: 6392.507] [val/train_update_time: 3570.813] [val/loss: 4.316] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.482] [val/val_tokens_per_second: 452687.798] [val/loss_avg_len_2048: 4.316] [val/perplexity_len_2048: 74.856] [val/loss_avg_len_1024: 4.339] [val/perplexity_len_1024: 76.654] [val/loss_avg_len_512: 4.382] [val/perplexity_len_512: 80.018]
+[2025-10-28 19:22:59][utils:57][INFO] [P: 65.00%] [S: 1363148800/2097152000] [T: 1:48:58] [ETA: 0:58:40] [loss: 4.303] [tokens/s: 206349.261] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:22:59][logger:171][INFO] [step: 1363148800] [train_eval/train_token_count: 1363148800] [train_eval/train_batch_count: 650] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 6538.856] [train_eval/train_update_time: 3626.554] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.332] [train_eval/perplexity_len_2048: 76.080] [train_eval/loss_avg_len_1024: 4.354] [train_eval/perplexity_len_1024: 77.764] [train_eval/loss_avg_len_512: 4.393] [train_eval/perplexity_len_512: 80.909]
+[2025-10-28 19:23:55][utils:57][INFO] [P: 66.00%] [S: 1384120320/2097152000] [T: 1:49:54] [ETA: 0:56:37] [loss: 4.325] [tokens/s: 226741.379] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:23:55][train:194][INFO] Running validation...
+[2025-10-28 19:25:26][logger:171][INFO] [step: 1384120320] [val/train_token_count: 1384120320] [val/train_batch_count: 660] [val/train_flop_count: 0] [val/train_total_time: 6594.719] [val/train_update_time: 3682.305] [val/loss: 4.299] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.584] [val/val_tokens_per_second: 452179.132] [val/loss_avg_len_2048: 4.299] [val/perplexity_len_2048: 73.653] [val/loss_avg_len_1024: 4.324] [val/perplexity_len_1024: 75.478] [val/loss_avg_len_512: 4.368] [val/perplexity_len_512: 78.894]
+[2025-10-28 19:26:22][utils:57][INFO] [P: 67.00%] [S: 1405091840/2097152000] [T: 1:52:21] [ETA: 0:55:20] [loss: 4.268] [tokens/s: 206324.362] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:27:18][utils:57][INFO] [P: 68.00%] [S: 1426063360/2097152000] [T: 1:53:17] [ETA: 0:53:18] [loss: 4.283] [tokens/s: 226722.267] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:27:18][train:194][INFO] Running validation...
+[2025-10-28 19:28:48][logger:171][INFO] [step: 1426063360] [val/train_token_count: 1426063360] [val/train_batch_count: 680] [val/train_flop_count: 0] [val/train_total_time: 6797.037] [val/train_update_time: 3793.800] [val/loss: 4.282] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.482] [val/val_tokens_per_second: 452685.977] [val/loss_avg_len_2048: 4.282] [val/perplexity_len_2048: 72.367] [val/loss_avg_len_1024: 4.307] [val/perplexity_len_1024: 74.230] [val/loss_avg_len_512: 4.353] [val/perplexity_len_512: 77.716]
+[2025-10-28 19:29:44][utils:57][INFO] [P: 69.00%] [S: 1447034880/2097152000] [T: 1:55:43] [ETA: 0:51:59] [loss: 4.283] [tokens/s: 206324.071] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:30:40][utils:57][INFO] [P: 70.00%] [S: 1468006400/2097152000] [T: 1:56:39] [ETA: 0:49:59] [loss: 4.279] [tokens/s: 226900.231] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:30:40][logger:171][INFO] [step: 1468006400] [train_eval/train_token_count: 1468006400] [train_eval/train_batch_count: 700] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 6999.246] [train_eval/train_update_time: 3905.298] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.285] [train_eval/perplexity_len_2048: 72.618] [train_eval/loss_avg_len_1024: 4.308] [train_eval/perplexity_len_1024: 74.319] [train_eval/loss_avg_len_512: 4.353] [train_eval/perplexity_len_512: 77.722]
+[2025-10-28 19:30:40][train:194][INFO] Running validation...
+[2025-10-28 19:32:10][logger:171][INFO] [step: 1468006400] [val/train_token_count: 1468006400] [val/train_batch_count: 700] [val/train_flop_count: 0] [val/train_total_time: 6999.246] [val/train_update_time: 3905.298] [val/loss: 4.266] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.416] [val/val_tokens_per_second: 453016.768] [val/loss_avg_len_2048: 4.266] [val/perplexity_len_2048: 71.260] [val/loss_avg_len_1024: 4.292] [val/perplexity_len_1024: 73.148] [val/loss_avg_len_512: 4.340] [val/perplexity_len_512: 76.683]
+[2025-10-28 19:32:10][checkpoint:111][INFO] Saving checkpoint to /workspace/forgetting-transformer/alibi_2_4_256/checkpoints/step-000001468006400.pt...
+[2025-10-28 19:32:11][checkpoint:128][INFO] Checkpoint saved to /workspace/forgetting-transformer/alibi_2_4_256/checkpoints/step-000001468006400.pt.
+[2025-10-28 19:32:11][logger:171][INFO] [step: 1468006400] [checkpoint/checkpoint_time: 0.443]
+[2025-10-28 19:33:07][utils:57][INFO] [P: 71.00%] [S: 1488977920/2097152000] [T: 1:59:05] [ETA: 0:48:38] [loss: 4.275] [tokens/s: 206397.827] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:34:02][utils:57][INFO] [P: 72.00%] [S: 1509949440/2097152000] [T: 2:00:01] [ETA: 0:46:40] [loss: 4.251] [tokens/s: 226785.687] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:34:02][train:194][INFO] Running validation...
+[2025-10-28 19:35:33][logger:171][INFO] [step: 1509949440] [val/train_token_count: 1509949440] [val/train_batch_count: 720] [val/train_flop_count: 0] [val/train_total_time: 7201.856] [val/train_update_time: 4016.813] [val/loss: 4.252] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.413] [val/val_tokens_per_second: 453034.368] [val/loss_avg_len_2048: 4.252] [val/perplexity_len_2048: 70.277] [val/loss_avg_len_1024: 4.280] [val/perplexity_len_1024: 72.211] [val/loss_avg_len_512: 4.328] [val/perplexity_len_512: 75.819]
+[2025-10-28 19:36:29][utils:57][INFO] [P: 73.00%] [S: 1530920960/2097152000] [T: 2:02:28] [ETA: 0:45:17] [loss: 4.268] [tokens/s: 206403.190] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:37:25][utils:57][INFO] [P: 74.00%] [S: 1551892480/2097152000] [T: 2:03:24] [ETA: 0:43:21] [loss: 4.253] [tokens/s: 226803.239] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:37:25][train:194][INFO] Running validation...
+[2025-10-28 19:38:55][logger:171][INFO] [step: 1551892480] [val/train_token_count: 1551892480] [val/train_batch_count: 740] [val/train_flop_count: 0] [val/train_total_time: 7404.001] [val/train_update_time: 4128.299] [val/loss: 4.240] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.461] [val/val_tokens_per_second: 452790.513] [val/loss_avg_len_2048: 4.240] [val/perplexity_len_2048: 69.406] [val/loss_avg_len_1024: 4.268] [val/perplexity_len_1024: 71.360] [val/loss_avg_len_512: 4.318] [val/perplexity_len_512: 75.015]
+[2025-10-28 19:39:51][utils:57][INFO] [P: 75.00%] [S: 1572864000/2097152000] [T: 2:05:50] [ETA: 0:41:56] [loss: 4.235] [tokens/s: 206405.713] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:39:51][logger:171][INFO] [step: 1572864000] [train_eval/train_token_count: 1572864000] [train_eval/train_batch_count: 750] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 7550.324] [train_eval/train_update_time: 4184.042] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.248] [train_eval/perplexity_len_2048: 69.957] [train_eval/loss_avg_len_1024: 4.272] [train_eval/perplexity_len_1024: 71.697] [train_eval/loss_avg_len_512: 4.320] [train_eval/perplexity_len_512: 75.223]
+[2025-10-28 19:40:47][utils:57][INFO] [P: 76.00%] [S: 1593835520/2097152000] [T: 2:06:46] [ETA: 0:40:01] [loss: 4.197] [tokens/s: 226835.759] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:40:47][train:194][INFO] Running validation...
+[2025-10-28 19:42:17][logger:171][INFO] [step: 1593835520] [val/train_token_count: 1593835520] [val/train_batch_count: 760] [val/train_flop_count: 0] [val/train_total_time: 7606.182] [val/train_update_time: 4239.792] [val/loss: 4.229] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.436] [val/val_tokens_per_second: 452919.183] [val/loss_avg_len_2048: 4.229] [val/perplexity_len_2048: 68.645] [val/loss_avg_len_1024: 4.258] [val/perplexity_len_1024: 70.649] [val/loss_avg_len_512: 4.309] [val/perplexity_len_512: 74.386]
+[2025-10-28 19:43:13][utils:57][INFO] [P: 77.00%] [S: 1614807040/2097152000] [T: 2:09:12] [ETA: 0:38:35] [loss: 4.262] [tokens/s: 206438.318] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:44:09][utils:57][INFO] [P: 78.00%] [S: 1635778560/2097152000] [T: 2:10:08] [ETA: 0:36:42] [loss: 4.183] [tokens/s: 226855.893] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:44:09][train:194][INFO] Running validation...
+[2025-10-28 19:45:39][logger:171][INFO] [step: 1635778560] [val/train_token_count: 1635778560] [val/train_batch_count: 780] [val/train_flop_count: 0] [val/train_total_time: 7808.319] [val/train_update_time: 4351.274] [val/loss: 4.219] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.456] [val/val_tokens_per_second: 452816.934] [val/loss_avg_len_2048: 4.219] [val/perplexity_len_2048: 67.958] [val/loss_avg_len_1024: 4.248] [val/perplexity_len_1024: 69.968] [val/loss_avg_len_512: 4.300] [val/perplexity_len_512: 73.721]
+[2025-10-28 19:46:35][utils:57][INFO] [P: 79.00%] [S: 1656750080/2097152000] [T: 2:12:34] [ETA: 0:35:14] [loss: 4.212] [tokens/s: 206459.168] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:47:31][utils:57][INFO] [P: 80.00%] [S: 1677721600/2097152000] [T: 2:13:30] [ETA: 0:33:22] [loss: 4.190] [tokens/s: 226972.557] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:47:31][logger:171][INFO] [step: 1677721600] [train_eval/train_token_count: 1677721600] [train_eval/train_batch_count: 800] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 8010.438] [train_eval/train_update_time: 4462.723] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.221] [train_eval/perplexity_len_2048: 68.081] [train_eval/loss_avg_len_1024: 4.248] [train_eval/perplexity_len_1024: 69.977] [train_eval/loss_avg_len_512: 4.299] [train_eval/perplexity_len_512: 73.594]
+[2025-10-28 19:47:31][train:194][INFO] Running validation...
+[2025-10-28 19:49:02][logger:171][INFO] [step: 1677721600] [val/train_token_count: 1677721600] [val/train_batch_count: 800] [val/train_flop_count: 0] [val/train_total_time: 8010.438] [val/train_update_time: 4462.723] [val/loss: 4.211] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.428] [val/val_tokens_per_second: 452958.454] [val/loss_avg_len_2048: 4.211] [val/perplexity_len_2048: 67.393] [val/loss_avg_len_1024: 4.240] [val/perplexity_len_1024: 69.429] [val/loss_avg_len_512: 4.294] [val/perplexity_len_512: 73.234]
+[2025-10-28 19:49:02][checkpoint:111][INFO] Saving checkpoint to /workspace/forgetting-transformer/alibi_2_4_256/checkpoints/step-000001677721600.pt...
+[2025-10-28 19:49:02][checkpoint:128][INFO] Checkpoint saved to /workspace/forgetting-transformer/alibi_2_4_256/checkpoints/step-000001677721600.pt.
+[2025-10-28 19:49:02][logger:171][INFO] [step: 1677721600] [checkpoint/checkpoint_time: 0.446]
+[2025-10-28 19:49:58][utils:57][INFO] [P: 81.00%] [S: 1698693120/2097152000] [T: 2:15:57] [ETA: 0:31:53] [loss: 4.166] [tokens/s: 206463.686] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:50:54][utils:57][INFO] [P: 82.00%] [S: 1719664640/2097152000] [T: 2:16:52] [ETA: 0:30:02] [loss: 4.178] [tokens/s: 226873.613] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:50:54][train:194][INFO] Running validation...
+[2025-10-28 19:52:24][logger:171][INFO] [step: 1719664640] [val/train_token_count: 1719664640] [val/train_batch_count: 820] [val/train_flop_count: 0] [val/train_total_time: 8212.996] [val/train_update_time: 4574.181] [val/loss: 4.203] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.447] [val/val_tokens_per_second: 452860.995] [val/loss_avg_len_2048: 4.203] [val/perplexity_len_2048: 66.881] [val/loss_avg_len_1024: 4.233] [val/perplexity_len_1024: 68.921] [val/loss_avg_len_512: 4.287] [val/perplexity_len_512: 72.739]
+[2025-10-28 19:53:20][utils:57][INFO] [P: 83.00%] [S: 1740636160/2097152000] [T: 2:19:19] [ETA: 0:28:32] [loss: 4.211] [tokens/s: 206466.983] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:54:16][utils:57][INFO] [P: 84.00%] [S: 1761607680/2097152000] [T: 2:20:15] [ETA: 0:26:42] [loss: 4.159] [tokens/s: 226881.555] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:54:16][train:194][INFO] Running validation...
+[2025-10-28 19:55:46][logger:171][INFO] [step: 1761607680] [val/train_token_count: 1761607680] [val/train_batch_count: 840] [val/train_flop_count: 0] [val/train_total_time: 8415.157] [val/train_update_time: 4685.668] [val/loss: 4.197] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.465] [val/val_tokens_per_second: 452771.529] [val/loss_avg_len_2048: 4.197] [val/perplexity_len_2048: 66.470] [val/loss_avg_len_1024: 4.227] [val/perplexity_len_1024: 68.523] [val/loss_avg_len_512: 4.282] [val/perplexity_len_512: 72.362]
+[2025-10-28 19:56:42][utils:57][INFO] [P: 85.00%] [S: 1782579200/2097152000] [T: 2:22:41] [ETA: 0:25:10] [loss: 4.228] [tokens/s: 206470.796] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:56:42][logger:171][INFO] [step: 1782579200] [train_eval/train_token_count: 1782579200] [train_eval/train_batch_count: 850] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 8561.476] [train_eval/train_update_time: 4741.406] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.194] [train_eval/perplexity_len_2048: 66.266] [train_eval/loss_avg_len_1024: 4.217] [train_eval/perplexity_len_1024: 67.816] [train_eval/loss_avg_len_512: 4.269] [train_eval/perplexity_len_512: 71.476]
+[2025-10-28 19:57:38][utils:57][INFO] [P: 86.00%] [S: 1803550720/2097152000] [T: 2:23:37] [ETA: 0:23:22] [loss: 4.208] [tokens/s: 226877.197] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 19:57:38][train:194][INFO] Running validation...
+[2025-10-28 19:59:08][logger:171][INFO] [step: 1803550720] [val/train_token_count: 1803550720] [val/train_batch_count: 860] [val/train_flop_count: 0] [val/train_total_time: 8617.329] [val/train_update_time: 4797.151] [val/loss: 4.192] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.459] [val/val_tokens_per_second: 452800.632] [val/loss_avg_len_2048: 4.192] [val/perplexity_len_2048: 66.143] [val/loss_avg_len_1024: 4.223] [val/perplexity_len_1024: 68.206] [val/loss_avg_len_512: 4.278] [val/perplexity_len_512: 72.068]
+[2025-10-28 20:00:04][utils:57][INFO] [P: 87.00%] [S: 1824522240/2097152000] [T: 2:26:03] [ETA: 0:21:49] [loss: 4.153] [tokens/s: 206464.190] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 20:01:00][utils:57][INFO] [P: 88.00%] [S: 1845493760/2097152000] [T: 2:26:59] [ETA: 0:20:02] [loss: 4.171] [tokens/s: 226870.062] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 20:01:00][train:194][INFO] Running validation...
+[2025-10-28 20:02:31][logger:171][INFO] [step: 1845493760] [val/train_token_count: 1845493760] [val/train_batch_count: 880] [val/train_flop_count: 0] [val/train_total_time: 8819.515] [val/train_update_time: 4908.649] [val/loss: 4.188] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.859] [val/val_tokens_per_second: 450806.261] [val/loss_avg_len_2048: 4.188] [val/perplexity_len_2048: 65.883] [val/loss_avg_len_1024: 4.219] [val/perplexity_len_1024: 67.948] [val/loss_avg_len_512: 4.274] [val/perplexity_len_512: 71.819]
+[2025-10-28 20:03:27][utils:57][INFO] [P: 89.00%] [S: 1866465280/2097152000] [T: 2:29:26] [ETA: 0:18:28] [loss: 4.226] [tokens/s: 206367.168] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 20:04:23][utils:57][INFO] [P: 90.00%] [S: 1887436800/2097152000] [T: 2:30:22] [ETA: 0:16:42] [loss: 4.142] [tokens/s: 226851.795] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 20:04:23][logger:171][INFO] [step: 1887436800] [train_eval/train_token_count: 1887436800] [train_eval/train_batch_count: 900] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 9022.127] [train_eval/train_update_time: 5020.160] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.187] [train_eval/perplexity_len_2048: 65.827] [train_eval/loss_avg_len_1024: 4.214] [train_eval/perplexity_len_1024: 67.593] [train_eval/loss_avg_len_512: 4.267] [train_eval/perplexity_len_512: 71.341]
+[2025-10-28 20:04:23][train:194][INFO] Running validation...
+[2025-10-28 20:05:53][logger:171][INFO] [step: 1887436800] [val/train_token_count: 1887436800] [val/train_batch_count: 900] [val/train_flop_count: 0] [val/train_total_time: 9022.127] [val/train_update_time: 5020.160] [val/loss: 4.185] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.705] [val/val_tokens_per_second: 451573.310] [val/loss_avg_len_2048: 4.185] [val/perplexity_len_2048: 65.690] [val/loss_avg_len_1024: 4.216] [val/perplexity_len_1024: 67.765] [val/loss_avg_len_512: 4.272] [val/perplexity_len_512: 71.654]
+[2025-10-28 20:05:53][checkpoint:111][INFO] Saving checkpoint to /workspace/forgetting-transformer/alibi_2_4_256/checkpoints/step-000001887436800.pt...
+[2025-10-28 20:05:54][checkpoint:128][INFO] Checkpoint saved to /workspace/forgetting-transformer/alibi_2_4_256/checkpoints/step-000001887436800.pt.
+[2025-10-28 20:05:54][logger:171][INFO] [step: 1887436800] [checkpoint/checkpoint_time: 0.440]
+[2025-10-28 20:06:50][utils:57][INFO] [P: 91.00%] [S: 1908408320/2097152000] [T: 2:32:49] [ETA: 0:15:06] [loss: 4.176] [tokens/s: 206302.742] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 20:07:46][utils:57][INFO] [P: 92.00%] [S: 1929379840/2097152000] [T: 2:33:44] [ETA: 0:13:22] [loss: 4.205] [tokens/s: 226670.892] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 20:07:46][train:194][INFO] Running validation...
+[2025-10-28 20:09:16][logger:171][INFO] [step: 1929379840] [val/train_token_count: 1929379840] [val/train_batch_count: 920] [val/train_flop_count: 0] [val/train_total_time: 9224.990] [val/train_update_time: 5131.661] [val/loss: 4.183] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.654] [val/val_tokens_per_second: 451825.741] [val/loss_avg_len_2048: 4.183] [val/perplexity_len_2048: 65.552] [val/loss_avg_len_1024: 4.214] [val/perplexity_len_1024: 67.634] [val/loss_avg_len_512: 4.270] [val/perplexity_len_512: 71.538]
+[2025-10-28 20:10:12][utils:57][INFO] [P: 93.00%] [S: 1950351360/2097152000] [T: 2:36:11] [ETA: 0:11:45] [loss: 4.189] [tokens/s: 206250.526] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 20:11:08][utils:57][INFO] [P: 94.00%] [S: 1971322880/2097152000] [T: 2:37:07] [ETA: 0:10:01] [loss: 4.146] [tokens/s: 226618.395] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 20:11:08][train:194][INFO] Running validation...
+[2025-10-28 20:12:39][logger:171][INFO] [step: 1971322880] [val/train_token_count: 1971322880] [val/train_batch_count: 940] [val/train_flop_count: 0] [val/train_total_time: 9427.380] [val/train_update_time: 5243.172] [val/loss: 4.182] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.673] [val/val_tokens_per_second: 451732.003] [val/loss_avg_len_2048: 4.182] [val/perplexity_len_2048: 65.464] [val/loss_avg_len_1024: 4.213] [val/perplexity_len_1024: 67.548] [val/loss_avg_len_512: 4.269] [val/perplexity_len_512: 71.453]
+[2025-10-28 20:13:35][utils:57][INFO] [P: 95.00%] [S: 1992294400/2097152000] [T: 2:39:33] [ETA: 0:08:23] [loss: 4.169] [tokens/s: 206203.646] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 20:13:35][logger:171][INFO] [step: 1992294400] [train_eval/train_token_count: 1992294400] [train_eval/train_batch_count: 950] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 9573.927] [train_eval/train_update_time: 5298.921] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.175] [train_eval/perplexity_len_2048: 65.014] [train_eval/loss_avg_len_1024: 4.204] [train_eval/perplexity_len_1024: 66.984] [train_eval/loss_avg_len_512: 4.258] [train_eval/perplexity_len_512: 70.662]
+[2025-10-28 20:14:30][utils:57][INFO] [P: 96.00%] [S: 2013265920/2097152000] [T: 2:40:29] [ETA: 0:06:41] [loss: 4.167] [tokens/s: 226555.965] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 20:14:30][train:194][INFO] Running validation...
+[2025-10-28 20:16:01][logger:171][INFO] [step: 2013265920] [val/train_token_count: 2013265920] [val/train_batch_count: 960] [val/train_flop_count: 0] [val/train_total_time: 9629.799] [val/train_update_time: 5354.673] [val/loss: 4.181] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.827] [val/val_tokens_per_second: 450966.458] [val/loss_avg_len_2048: 4.181] [val/perplexity_len_2048: 65.413] [val/loss_avg_len_1024: 4.212] [val/perplexity_len_1024: 67.495] [val/loss_avg_len_512: 4.268] [val/perplexity_len_512: 71.400]
+[2025-10-28 20:16:57][utils:57][INFO] [P: 97.00%] [S: 2034237440/2097152000] [T: 2:42:56] [ETA: 0:05:02] [loss: 4.191] [tokens/s: 206126.025] [batches/s: 0.098] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 20:17:53][utils:57][INFO] [P: 98.00%] [S: 2055208960/2097152000] [T: 2:43:52] [ETA: 0:03:20] [loss: 4.167] [tokens/s: 226569.337] [batches/s: 0.108] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-28 20:17:53][train:194][INFO] Running validation...
+[2025-10-28 20:19:24][logger:171][INFO] [step: 2055208960] [val/train_token_count: 2055208960] [val/train_batch_count: 980] [val/train_flop_count: 0] [val/train_total_time: 9832.331] [val/train_update_time: 5466.160] [val/loss: 4.180] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 90.653] [val/val_tokens_per_second: 451832.038] [val/loss_avg_len_2048: 4.180] [val/perplexity_len_2048: 65.391] [val/loss_avg_len_1024: 4.212] [val/perplexity_len_1024: 67.476] [val/loss_avg_len_512: 4.268] [val/perplexity_len_512: 71.383]
+[2025-10-28 20:19:24][train:854][INFO] Training finished with 2055208960 tokens!

metrics/jsonlines/checkpoint.jsonl

@@ -0,0 +1,9 @@
+{"step": 209715200, "checkpoint/checkpoint_time": 0.45219888899009675}
+{"step": 419430400, "checkpoint/checkpoint_time": 0.44284954003524035}
+{"step": 629145600, "checkpoint/checkpoint_time": 0.4364313690457493}
+{"step": 838860800, "checkpoint/checkpoint_time": 0.44323001499287784}
+{"step": 1048576000, "checkpoint/checkpoint_time": 0.43819961103145033}
+{"step": 1258291200, "checkpoint/checkpoint_time": 0.4460486189927906}
+{"step": 1468006400, "checkpoint/checkpoint_time": 0.4434796510031447}
+{"step": 1677721600, "checkpoint/checkpoint_time": 0.4459765850333497}
+{"step": 1887436800, "checkpoint/checkpoint_time": 0.439551091985777}

metrics/jsonlines/model_info.jsonl

@@ -0,0 +1 @@
+{"step": 0, "model_info/total_params": 27447040, "model_info/trainable_params": 27447040, "model_info/embedding_params": 12870912, "model_info/flops_per_token": 0, "model_info/non_embedding_params": 14576128}

metrics/jsonlines/resume.jsonl

@@ -0,0 +1 @@
+{"step": 0, "resume/resume_step": 0}

metrics/jsonlines/train.jsonl

@@ -0,0 +1,98 @@
+{"step": 20971520, "train/token_count": 20971520, "train/batch_count": 10, "train/flop_count": 0, "train/total_time": 59.51673407689668, "train/update_time": 59.329752838937566, "train/lr": 0.0009000000000000001, "train/loss": 9.761818885803223, "train/global_grad_norm": 1.2346482276916504}
+{"step": 41943040, "train/token_count": 41943040, "train/batch_count": 20, "train/flop_count": 0, "train/total_time": 115.38901872490533, "train/update_time": 115.08477980294265, "train/lr": 0.0009997960964140947, "train/loss": 8.126626968383789, "train/global_grad_norm": 0.9628385305404663}
+{"step": 62914560, "train/token_count": 62914560, "train/batch_count": 30, "train/flop_count": 0, "train/total_time": 262.1596394549124, "train/update_time": 170.83132880181074, "train/lr": 0.0009990914580222257, "train/loss": 7.51987886428833, "train/global_grad_norm": 0.570928692817688}
+{"step": 83886080, "train/token_count": 83886080, "train/batch_count": 40, "train/flop_count": 0, "train/total_time": 318.01136298198253, "train/update_time": 226.56994250579737, "train/lr": 0.0009978842768382998, "train/loss": 7.193209648132324, "train/global_grad_norm": 0.4207130968570709}
+{"step": 104857600, "train/token_count": 104857600, "train/batch_count": 50, "train/flop_count": 0, "train/total_time": 465.39904637495056, "train/update_time": 282.29461103421636, "train/lr": 0.0009961757683914405, "train/loss": 6.9471588134765625, "train/global_grad_norm": 0.2690547704696655}
+{"step": 125829120, "train/token_count": 125829120, "train/batch_count": 60, "train/flop_count": 0, "train/total_time": 521.2494050179375, "train/update_time": 338.0269747101702, "train/lr": 0.00099396765300483, "train/loss": 6.683192729949951, "train/global_grad_norm": 0.39732715487480164}
+{"step": 146800640, "train/token_count": 146800640, "train/batch_count": 70, "train/flop_count": 0, "train/total_time": 667.8293691409053, "train/update_time": 393.75072771217674, "train/lr": 0.0009912621540634887, "train/loss": 6.482468128204346, "train/global_grad_norm": 0.30048173666000366}
+{"step": 167772160, "train/token_count": 167772160, "train/batch_count": 80, "train/flop_count": 0, "train/total_time": 723.6805678269593, "train/update_time": 449.4893446461065, "train/lr": 0.000988061995775515, "train/loss": 6.281726837158203, "train/global_grad_norm": 0.3598792552947998}
+{"step": 188743680, "train/token_count": 188743680, "train/batch_count": 90, "train/flop_count": 0, "train/total_time": 871.0406558898976, "train/update_time": 505.2065281631658, "train/lr": 0.0009843704004290394, "train/loss": 6.091277122497559, "train/global_grad_norm": 0.38550785183906555}
+{"step": 209715200, "train/token_count": 209715200, "train/batch_count": 100, "train/flop_count": 0, "train/total_time": 926.8863875919487, "train/update_time": 560.9345708230976, "train/lr": 0.0009801910851476522, "train/loss": 5.968027114868164, "train/global_grad_norm": 0.38172265887260437}
+{"step": 230686720, "train/token_count": 230686720, "train/batch_count": 110, "train/flop_count": 0, "train/total_time": 1073.8704686219571, "train/update_time": 616.6548248290783, "train/lr": 0.0009755282581475768, "train/loss": 5.854724407196045, "train/global_grad_norm": 0.6797294020652771}
+{"step": 251658240, "train/token_count": 251658240, "train/batch_count": 120, "train/flop_count": 0, "train/total_time": 1129.7145008929074, "train/update_time": 672.3896088181064, "train/lr": 0.0009703866145003512, "train/loss": 5.713388442993164, "train/global_grad_norm": 0.4913008511066437}
+{"step": 272629760, "train/token_count": 272629760, "train/batch_count": 130, "train/flop_count": 0, "train/total_time": 1276.169682028005, "train/update_time": 728.117079011281, "train/lr": 0.0009647713314052896, "train/loss": 5.644865036010742, "train/global_grad_norm": 0.3544484078884125}
+{"step": 293601280, "train/token_count": 293601280, "train/batch_count": 140, "train/flop_count": 0, "train/total_time": 1332.0184532779967, "train/update_time": 783.8552743501496, "train/lr": 0.0009586880629764817, "train/loss": 5.568542957305908, "train/global_grad_norm": 0.49100813269615173}
+{"step": 314572800, "train/token_count": 314572800, "train/batch_count": 150, "train/flop_count": 0, "train/total_time": 1478.8752941149287, "train/update_time": 839.5983157731825, "train/lr": 0.0009521429345495787, "train/loss": 5.447315216064453, "train/global_grad_norm": 0.36811110377311707}
+{"step": 335544320, "train/token_count": 335544320, "train/batch_count": 160, "train/flop_count": 0, "train/total_time": 1534.7253116948996, "train/update_time": 895.3389847053913, "train/lr": 0.0009451425365140996, "train/loss": 5.412559986114502, "train/global_grad_norm": 0.8786026239395142}
+{"step": 356515840, "train/token_count": 356515840, "train/batch_count": 170, "train/flop_count": 0, "train/total_time": 1681.113392906962, "train/update_time": 951.0781835562084, "train/lr": 0.000937693917677468, "train/loss": 5.300525188446045, "train/global_grad_norm": 0.37146928906440735}
+{"step": 377487360, "train/token_count": 377487360, "train/batch_count": 180, "train/flop_count": 0, "train/total_time": 1736.965916060959, "train/update_time": 1006.818993799272, "train/lr": 0.0009298045781674596, "train/loss": 5.27311372756958, "train/global_grad_norm": 0.6399803161621094}
+{"step": 398458880, "train/token_count": 398458880, "train/batch_count": 190, "train/flop_count": 0, "train/total_time": 1883.3845960129984, "train/update_time": 1062.5558878729353, "train/lr": 0.0009214824618802108, "train/loss": 5.24492073059082, "train/global_grad_norm": 0.5269308090209961}
+{"step": 419430400, "train/token_count": 419430400, "train/batch_count": 200, "train/flop_count": 0, "train/total_time": 1939.2464721049182, "train/update_time": 1118.2928310850402, "train/lr": 0.000912735948481387, "train/loss": 5.150586128234863, "train/global_grad_norm": 0.5451287031173706}
+{"step": 440401920, "train/token_count": 440401920, "train/batch_count": 210, "train/flop_count": 0, "train/total_time": 2086.0689364429563, "train/update_time": 1174.0314252841054, "train/lr": 0.0009035738449685707, "train/loss": 5.106629848480225, "train/global_grad_norm": 0.4432642161846161}
+{"step": 461373440, "train/token_count": 461373440, "train/batch_count": 220, "train/flop_count": 0, "train/total_time": 2141.922117186943, "train/update_time": 1229.7717256471515, "train/lr": 0.0008940053768033609, "train/loss": 5.072513103485107, "train/global_grad_norm": 0.4362945556640625}
+{"step": 482344960, "train/token_count": 482344960, "train/batch_count": 230, "train/flop_count": 0, "train/total_time": 2288.3046315449756, "train/update_time": 1285.5049123089993, "train/lr": 0.0008840401786221159, "train/loss": 5.0194478034973145, "train/global_grad_norm": 0.5716229677200317}
+{"step": 503316480, "train/token_count": 503316480, "train/batch_count": 240, "train/flop_count": 0, "train/total_time": 2344.147989144898, "train/update_time": 1341.2354510270525, "train/lr": 0.0008736882845346905, "train/loss": 4.964856147766113, "train/global_grad_norm": 0.4435354769229889}
+{"step": 524288000, "train/token_count": 524288000, "train/batch_count": 250, "train/flop_count": 0, "train/total_time": 2491.0224517869065, "train/update_time": 1396.97127348301, "train/lr": 0.0008629601180209381, "train/loss": 4.966379165649414, "train/global_grad_norm": 0.7064673900604248}
+{"step": 545259520, "train/token_count": 545259520, "train/batch_count": 260, "train/flop_count": 0, "train/total_time": 2546.883552026935, "train/update_time": 1452.7241989910835, "train/lr": 0.0008518664814351503, "train/loss": 4.9135613441467285, "train/global_grad_norm": 0.5128141045570374}
+{"step": 566231040, "train/token_count": 566231040, "train/batch_count": 270, "train/flop_count": 0, "train/total_time": 2693.2529326969525, "train/update_time": 1508.4631838970818, "train/lr": 0.0008404185451290017, "train/loss": 4.897538661956787, "train/global_grad_norm": 0.376214861869812}
+{"step": 587202560, "train/token_count": 587202560, "train/batch_count": 280, "train/flop_count": 0, "train/total_time": 2749.1005825489992, "train/update_time": 1564.2029820160242, "train/lr": 0.0008286278362039527, "train/loss": 4.849531173706055, "train/global_grad_norm": 0.5354404449462891}
+{"step": 608174080, "train/token_count": 608174080, "train/batch_count": 290, "train/flop_count": 0, "train/total_time": 2895.4641819539247, "train/update_time": 1619.943224380957, "train/lr": 0.0008165062269044352, "train/loss": 4.818095684051514, "train/global_grad_norm": 0.47904765605926514}
+{"step": 629145600, "train/token_count": 629145600, "train/batch_count": 300, "train/flop_count": 0, "train/total_time": 2951.315687590977, "train/update_time": 1675.687991371029, "train/lr": 0.0008040659226635089, "train/loss": 4.798238277435303, "train/global_grad_norm": 0.6568677425384521}
+{"step": 650117120, "train/token_count": 650117120, "train/batch_count": 310, "train/flop_count": 0, "train/total_time": 3098.1222598329186, "train/update_time": 1731.4270116091939, "train/lr": 0.0007913194498130252, "train/loss": 4.810272216796875, "train/global_grad_norm": 0.5013900399208069}
+{"step": 671088640, "train/token_count": 671088640, "train/batch_count": 320, "train/flop_count": 0, "train/total_time": 3153.9731954459567, "train/update_time": 1787.1771466861246, "train/lr": 0.000778279642970672, "train/loss": 4.744162559509277, "train/global_grad_norm": 0.5148316621780396}
+{"step": 692060160, "train/token_count": 692060160, "train/batch_count": 330, "train/flop_count": 0, "train/total_time": 3300.597545653931, "train/update_time": 1842.9211868423736, "train/lr": 0.0007649596321166025, "train/loss": 4.759054660797119, "train/global_grad_norm": 0.4511776864528656}
+{"step": 713031680, "train/token_count": 713031680, "train/batch_count": 340, "train/flop_count": 0, "train/total_time": 3356.450910591986, "train/update_time": 1898.6706604874926, "train/lr": 0.0007513728293726579, "train/loss": 4.724034309387207, "train/global_grad_norm": 0.5304577946662903}
+{"step": 734003200, "train/token_count": 734003200, "train/batch_count": 350, "train/flop_count": 0, "train/total_time": 3502.8775023539783, "train/update_time": 1954.4126074366504, "train/lr": 0.0007375329154974975, "train/loss": 4.703649044036865, "train/global_grad_norm": 0.4927815794944763}
+{"step": 754974720, "train/token_count": 754974720, "train/batch_count": 360, "train/flop_count": 0, "train/total_time": 3558.7265775619308, "train/update_time": 2010.1547237539198, "train/lr": 0.0007234538261112341, "train/loss": 4.634231090545654, "train/global_grad_norm": 0.5631891489028931}
+{"step": 775946240, "train/token_count": 775946240, "train/batch_count": 370, "train/flop_count": 0, "train/total_time": 3705.2644805479795, "train/update_time": 2065.915084464941, "train/lr": 0.0007091497376634464, "train/loss": 4.656914234161377, "train/global_grad_norm": 0.5161179304122925}
+{"step": 796917760, "train/token_count": 796917760, "train/batch_count": 380, "train/flop_count": 0, "train/total_time": 3761.109741097898, "train/update_time": 2121.6608164007775, "train/lr": 0.0006946350531586958, "train/loss": 4.634555339813232, "train/global_grad_norm": 0.5455919504165649}
+{"step": 817889280, "train/token_count": 817889280, "train/batch_count": 390, "train/flop_count": 0, "train/total_time": 3907.6489149519475, "train/update_time": 2177.411766545032, "train/lr": 0.0006799243876539214, "train/loss": 4.638850688934326, "train/global_grad_norm": 0.5190625190734863}
+{"step": 838860800, "train/token_count": 838860800, "train/batch_count": 400, "train/flop_count": 0, "train/total_time": 3963.4901722619543, "train/update_time": 2233.1504810712067, "train/lr": 0.0006650325535423166, "train/loss": 4.546455383300781, "train/global_grad_norm": 0.47769442200660706}
+{"step": 859832320, "train/token_count": 859832320, "train/batch_count": 410, "train/flop_count": 0, "train/total_time": 4110.402928694966, "train/update_time": 2288.885604837211, "train/lr": 0.0006499745456385053, "train/loss": 4.568233489990234, "train/global_grad_norm": 0.5308087468147278}
+{"step": 880803840, "train/token_count": 880803840, "train/batch_count": 420, "train/flop_count": 0, "train/total_time": 4166.242571576964, "train/update_time": 2344.6231456701644, "train/lr": 0.0006347655260800339, "train/loss": 4.565241813659668, "train/global_grad_norm": 0.5333735942840576}
+{"step": 901775360, "train/token_count": 901775360, "train/batch_count": 430, "train/flop_count": 0, "train/total_time": 4312.675729307928, "train/update_time": 2400.370607822202, "train/lr": 0.0006194208090603844, "train/loss": 4.559530258178711, "train/global_grad_norm": 0.4546603262424469}
+{"step": 922746880, "train/token_count": 922746880, "train/batch_count": 440, "train/flop_count": 0, "train/total_time": 4368.526084997924, "train/update_time": 2456.1160258102464, "train/lr": 0.0006039558454088796, "train/loss": 4.5818610191345215, "train/global_grad_norm": 0.5177769660949707}
+{"step": 943718400, "train/token_count": 943718400, "train/batch_count": 450, "train/flop_count": 0, "train/total_time": 4515.121015235898, "train/update_time": 2511.850276188343, "train/lr": 0.0005883862070330078, "train/loss": 4.528136253356934, "train/global_grad_norm": 0.4562700092792511}
+{"step": 964689920, "train/token_count": 964689920, "train/batch_count": 460, "train/flop_count": 0, "train/total_time": 4570.982028090977, "train/update_time": 2567.596293253242, "train/lr": 0.0005727275712388317, "train/loss": 4.493772506713867, "train/global_grad_norm": 0.4743252992630005}
+{"step": 985661440, "train/token_count": 985661440, "train/batch_count": 470, "train/flop_count": 0, "train/total_time": 4717.434516418958, "train/update_time": 2623.332695179037, "train/lr": 0.0005569957049452703, "train/loss": 4.519767761230469, "train/global_grad_norm": 0.6404874324798584}
+{"step": 1006632960, "train/token_count": 1006632960, "train/batch_count": 480, "train/flop_count": 0, "train/total_time": 4773.292612125981, "train/update_time": 2679.07335356588, "train/lr": 0.0005412064488081482, "train/loss": 4.494389533996582, "train/global_grad_norm": 0.44306617975234985}
+{"step": 1027604480, "train/token_count": 1027604480, "train/batch_count": 490, "train/flop_count": 0, "train/total_time": 4919.765502344933, "train/update_time": 2734.8176154628163, "train/lr": 0.0005253757012699972, "train/loss": 4.489535331726074, "train/global_grad_norm": 0.45535698533058167}
+{"step": 1048576000, "train/token_count": 1048576000, "train/batch_count": 500, "train/flop_count": 0, "train/total_time": 4975.63103498891, "train/update_time": 2790.5577287059277, "train/lr": 0.0005095194025516734, "train/loss": 4.462835311889648, "train/global_grad_norm": 0.46494054794311523}
+{"step": 1069547520, "train/token_count": 1069547520, "train/batch_count": 510, "train/flop_count": 0, "train/total_time": 5122.565231077955, "train/update_time": 2846.292529964936, "train/lr": 0.0004936535186019053, "train/loss": 4.4626970291137695, "train/global_grad_norm": 0.42456820607185364}
+{"step": 1090519040, "train/token_count": 1090519040, "train/batch_count": 520, "train/flop_count": 0, "train/total_time": 5178.418338532909, "train/update_time": 2902.025444978848, "train/lr": 0.00047779402502093696, "train/loss": 4.453117370605469, "train/global_grad_norm": 0.47793683409690857}
+{"step": 1111490560, "train/token_count": 1111490560, "train/batch_count": 530, "train/flop_count": 0, "train/total_time": 5324.87164538994, "train/update_time": 2957.746671211091, "train/lr": 0.0004619568909744525, "train/loss": 4.413782119750977, "train/global_grad_norm": 0.5748726725578308}
+{"step": 1132462080, "train/token_count": 1132462080, "train/batch_count": 540, "train/flop_count": 0, "train/total_time": 5380.720167659922, "train/update_time": 3013.4729040842503, "train/lr": 0.00044615806311398067, "train/loss": 4.423956871032715, "train/global_grad_norm": 0.5001558661460876}
+{"step": 1153433600, "train/token_count": 1153433600, "train/batch_count": 550, "train/flop_count": 0, "train/total_time": 5527.108590850956, "train/update_time": 3069.2099333773367, "train/lr": 0.0004304134495199673, "train/loss": 4.370279788970947, "train/global_grad_norm": 0.5166635513305664}
+{"step": 1174405120, "train/token_count": 1174405120, "train/batch_count": 560, "train/flop_count": 0, "train/total_time": 5582.958667394938, "train/update_time": 3124.937779762433, "train/lr": 0.0004147389036836882, "train/loss": 4.411366939544678, "train/global_grad_norm": 0.4276511073112488}
+{"step": 1195376640, "train/token_count": 1195376640, "train/batch_count": 570, "train/flop_count": 0, "train/total_time": 5729.297242018976, "train/update_time": 3180.655704040313, "train/lr": 0.0003991502085441259, "train/loss": 4.3608832359313965, "train/global_grad_norm": 0.44203898310661316}
+{"step": 1216348160, "train/token_count": 1216348160, "train/batch_count": 580, "train/flop_count": 0, "train/total_time": 5785.135669226991, "train/update_time": 3236.387389887357, "train/lr": 0.0003836630605958888, "train/loss": 4.409422874450684, "train/global_grad_norm": 0.44176411628723145}
+{"step": 1237319680, "train/token_count": 1237319680, "train/batch_count": 590, "train/flop_count": 0, "train/total_time": 5931.5355288069695, "train/update_time": 3292.107882911223, "train/lr": 0.00036829305408417155, "train/loss": 4.389522552490234, "train/global_grad_norm": 0.354754239320755}
+{"step": 1258291200, "train/token_count": 1258291200, "train/batch_count": 600, "train/flop_count": 0, "train/total_time": 5987.366702231928, "train/update_time": 3347.8448207870824, "train/lr": 0.000353055665302672, "train/loss": 4.390128135681152, "train/global_grad_norm": 0.6542595028877258}
+{"step": 1279262720, "train/token_count": 1279262720, "train/batch_count": 610, "train/flop_count": 0, "train/total_time": 6134.471666511963, "train/update_time": 3403.5747455290984, "train/lr": 0.0003379662370102746, "train/loss": 4.354618549346924, "train/global_grad_norm": 0.4821139872074127}
+{"step": 1300234240, "train/token_count": 1300234240, "train/batch_count": 620, "train/flop_count": 0, "train/total_time": 6190.319736288977, "train/update_time": 3459.312256404897, "train/lr": 0.00032303996298219405, "train/loss": 4.3281426429748535, "train/global_grad_norm": 0.4146212339401245}
+{"step": 1321205760, "train/token_count": 1321205760, "train/batch_count": 630, "train/flop_count": 0, "train/total_time": 6336.646818120964, "train/update_time": 3515.0587793228915, "train/lr": 0.00030829187271113034, "train/loss": 4.3390960693359375, "train/global_grad_norm": 0.4168533980846405}
+{"step": 1342177280, "train/token_count": 1342177280, "train/batch_count": 640, "train/flop_count": 0, "train/total_time": 6392.506873650011, "train/update_time": 3570.8129316339036, "train/lr": 0.0002937368162738445, "train/loss": 4.329615592956543, "train/global_grad_norm": 0.4574236571788788}
+{"step": 1363148800, "train/token_count": 1363148800, "train/batch_count": 650, "train/flop_count": 0, "train/total_time": 6538.855644405005, "train/update_time": 3626.554348448757, "train/lr": 0.0002793894493783894, "train/loss": 4.302555561065674, "train/global_grad_norm": 0.41884133219718933}
+{"step": 1384120320, "train/token_count": 1384120320, "train/batch_count": 660, "train/flop_count": 0, "train/total_time": 6594.719191052951, "train/update_time": 3682.304503756459, "train/lr": 0.00026526421860705474, "train/loss": 4.3252763748168945, "train/global_grad_norm": 0.5183114409446716}
+{"step": 1405091840, "train/token_count": 1405091840, "train/batch_count": 670, "train/flop_count": 0, "train/total_time": 6741.161298968946, "train/update_time": 3738.0453335597413, "train/lr": 0.0002513753468698824, "train/loss": 4.268364429473877, "train/global_grad_norm": 0.4133932888507843}
+{"step": 1426063360, "train/token_count": 1426063360, "train/batch_count": 680, "train/flop_count": 0, "train/total_time": 6797.037191045936, "train/update_time": 3793.7997314956738, "train/lr": 0.00023773681908340283, "train/loss": 4.28262996673584, "train/global_grad_norm": 0.4412442445755005}
+{"step": 1447034880, "train/token_count": 1447034880, "train/batch_count": 690, "train/flop_count": 0, "train/total_time": 6943.39617751597, "train/update_time": 3849.5505883715814, "train/lr": 0.00022436236808900823, "train/loss": 4.283445358276367, "train/global_grad_norm": 0.3459267318248749}
+{"step": 1468006400, "train/token_count": 1468006400, "train/batch_count": 700, "train/flop_count": 0, "train/total_time": 6999.246100232936, "train/update_time": 3905.29758988251, "train/lr": 0.00021126546082514682, "train/loss": 4.279012203216553, "train/global_grad_norm": 0.3911832273006439}
+{"step": 1488977920, "train/token_count": 1488977920, "train/batch_count": 710, "train/flop_count": 0, "train/total_time": 7145.973933565896, "train/update_time": 3961.0490365702426, "train/lr": 0.00019845928476725522, "train/loss": 4.275318622589111, "train/global_grad_norm": 0.3508543074131012}
+{"step": 1509949440, "train/token_count": 1509949440, "train/batch_count": 720, "train/flop_count": 0, "train/total_time": 7201.8562647239305, "train/update_time": 4016.812845747103, "train/lr": 0.0001859567346490913, "train/loss": 4.250895023345947, "train/global_grad_norm": 0.38312408328056335}
+{"step": 1530920960, "train/token_count": 1530920960, "train/batch_count": 730, "train/flop_count": 0, "train/total_time": 7348.130815029959, "train/update_time": 4072.5528116449714, "train/lr": 0.00017377039947882782, "train/loss": 4.26820707321167, "train/global_grad_norm": 0.4197799861431122}
+{"step": 1551892480, "train/token_count": 1551892480, "train/batch_count": 740, "train/flop_count": 0, "train/total_time": 7404.001054160995, "train/update_time": 4128.298564241966, "train/lr": 0.00016191254986299043, "train/loss": 4.2526726722717285, "train/global_grad_norm": 0.33865150809288025}
+{"step": 1572864000, "train/token_count": 1572864000, "train/batch_count": 750, "train/flop_count": 0, "train/total_time": 7550.323625019984, "train/update_time": 4184.042193662957, "train/lr": 0.00015039512565099468, "train/loss": 4.2354736328125, "train/global_grad_norm": 0.3443734049797058}
+{"step": 1593835520, "train/token_count": 1593835520, "train/batch_count": 760, "train/flop_count": 0, "train/total_time": 7606.181935134926, "train/update_time": 4239.792019490269, "train/lr": 0.00013922972391273224, "train/loss": 4.196894645690918, "train/global_grad_norm": 0.37375685572624207}
+{"step": 1614807040, "train/token_count": 1614807040, "train/batch_count": 770, "train/flop_count": 0, "train/total_time": 7752.467774236924, "train/update_time": 4295.5320715362905, "train/lr": 0.00012842758726130281, "train/loss": 4.261632919311523, "train/global_grad_norm": 0.3186478614807129}
+{"step": 1635778560, "train/token_count": 1635778560, "train/batch_count": 780, "train/flop_count": 0, "train/total_time": 7808.319425186957, "train/update_time": 4351.274368522107, "train/lr": 0.00011799959253265679, "train/loss": 4.1832122802734375, "train/global_grad_norm": 0.3403545916080475}
+{"step": 1656750080, "train/token_count": 1656750080, "train/batch_count": 790, "train/flop_count": 0, "train/total_time": 7954.606591358897, "train/update_time": 4407.000014613499, "train/lr": 0.00010795623983354214, "train/loss": 4.212279319763184, "train/global_grad_norm": 0.3116688132286072}
+{"step": 1677721600, "train/token_count": 1677721600, "train/batch_count": 800, "train/flop_count": 0, "train/total_time": 8010.43765124795, "train/update_time": 4462.722712960327, "train/lr": 9.830764196878872e-05, "train/loss": 4.190274715423584, "train/global_grad_norm": 0.3231128454208374}
+{"step": 1698693120, "train/token_count": 1698693120, "train/batch_count": 810, "train/flop_count": 0, "train/total_time": 8157.16301853396, "train/update_time": 4518.44748669432, "train/lr": 8.906351425856951e-05, "train/loss": 4.165951251983643, "train/global_grad_norm": 0.29648908972740173}
+{"step": 1719664640, "train/token_count": 1719664640, "train/batch_count": 820, "train/flop_count": 0, "train/total_time": 8212.996373801958, "train/update_time": 4574.1805559834465, "train/lr": 8.02331647558977e-05, "train/loss": 4.1781415939331055, "train/global_grad_norm": 0.2801770269870758}
+{"step": 1740636160, "train/token_count": 1740636160, "train/batch_count": 830, "train/flop_count": 0, "train/total_time": 8359.302277855924, "train/update_time": 4629.922305631335, "train/lr": 7.182548487420554e-05, "train/loss": 4.2106523513793945, "train/global_grad_norm": 0.2964642345905304}
+{"step": 1761607680, "train/token_count": 1761607680, "train/batch_count": 840, "train/flop_count": 0, "train/total_time": 8415.15657795791, "train/update_time": 4685.668182700407, "train/lr": 6.384894043444556e-05, "train/loss": 4.159307479858398, "train/global_grad_norm": 0.2973722517490387}
+{"step": 1782579200, "train/token_count": 1782579200, "train/batch_count": 850, "train/flop_count": 0, "train/total_time": 8561.475588510977, "train/update_time": 4741.406085017603, "train/lr": 5.6311563140726166e-05, "train/loss": 4.228494167327881, "train/global_grad_norm": 0.2686772048473358}
+{"step": 1803550720, "train/token_count": 1803550720, "train/batch_count": 860, "train/flop_count": 0, "train/total_time": 8617.329412178951, "train/update_time": 4797.150553135551, "train/lr": 4.922094249306547e-05, "train/loss": 4.207626819610596, "train/global_grad_norm": 0.25773489475250244}
+{"step": 1824522240, "train/token_count": 1824522240, "train/batch_count": 870, "train/flop_count": 0, "train/total_time": 8763.651462373906, "train/update_time": 4852.897546026274, "train/lr": 4.2584218145409916e-05, "train/loss": 4.153430461883545, "train/global_grad_norm": 0.2575656473636627}
+{"step": 1845493760, "train/token_count": 1845493760, "train/batch_count": 880, "train/flop_count": 0, "train/total_time": 8819.515494428924, "train/update_time": 4908.649304769933, "train/lr": 3.6408072716606236e-05, "train/loss": 4.171026229858398, "train/global_grad_norm": 0.27688679099082947}
+{"step": 1866465280, "train/token_count": 1866465280, "train/batch_count": 890, "train/flop_count": 0, "train/total_time": 8966.25939859997, "train/update_time": 4964.410062348121, "train/lr": 3.069872506157217e-05, "train/loss": 4.226474285125732, "train/global_grad_norm": 0.26012977957725525}
+{"step": 1887436800, "train/token_count": 1887436800, "train/batch_count": 900, "train/flop_count": 0, "train/total_time": 9022.126939511974, "train/update_time": 5020.160163923283, "train/lr": 2.5461924009435368e-05, "train/loss": 4.141654014587402, "train/global_grad_norm": 0.25259116291999817}
+{"step": 1908408320, "train/token_count": 1908408320, "train/batch_count": 910, "train/flop_count": 0, "train/total_time": 9169.132314599934, "train/update_time": 5075.909138886142, "train/lr": 2.0702942574950812e-05, "train/loss": 4.176390647888184, "train/global_grad_norm": 0.2537875175476074}
+{"step": 1929379840, "train/token_count": 1929379840, "train/batch_count": 920, "train/flop_count": 0, "train/total_time": 9224.990330599947, "train/update_time": 5131.661126127117, "train/lr": 1.642657264902142e-05, "train/loss": 4.204928398132324, "train/global_grad_norm": 0.23280541598796844}
+{"step": 1950351360, "train/token_count": 1950351360, "train/batch_count": 930, "train/flop_count": 0, "train/total_time": 9371.526648397907, "train/update_time": 5187.4173517230665, "train/lr": 1.2637120173670358e-05, "train/loss": 4.189211845397949, "train/global_grad_norm": 0.22280284762382507}
+{"step": 1971322880, "train/token_count": 1971322880, "train/batch_count": 940, "train/flop_count": 0, "train/total_time": 9427.379897051957, "train/update_time": 5243.171735763899, "train/lr": 9.338400806321978e-06, "train/loss": 4.146426200866699, "train/global_grad_norm": 0.22602267563343048}
+{"step": 1992294400, "train/token_count": 1992294400, "train/batch_count": 950, "train/flop_count": 0, "train/total_time": 9573.927157147904, "train/update_time": 5298.920672355802, "train/lr": 6.533736077758867e-06, "train/loss": 4.1687397956848145, "train/global_grad_norm": 0.22272436320781708}
+{"step": 2013265920, "train/token_count": 2013265920, "train/batch_count": 960, "train/flop_count": 0, "train/total_time": 9629.798734208918, "train/update_time": 5354.673169916612, "train/lr": 4.2259500476214406e-06, "train/loss": 4.167335510253906, "train/global_grad_norm": 0.21395571529865265}
+{"step": 2034237440, "train/token_count": 2034237440, "train/batch_count": 970, "train/flop_count": 0, "train/total_time": 9776.485576101928, "train/update_time": 5410.419541045674, "train/lr": 2.417366460819359e-06, "train/loss": 4.191149711608887, "train/global_grad_norm": 0.21384510397911072}
+{"step": 2055208960, "train/token_count": 2055208960, "train/batch_count": 980, "train/flop_count": 0, "train/total_time": 9832.331430819933, "train/update_time": 5466.160342909512, "train/lr": 1.1098064077174619e-06, "train/loss": 4.1665802001953125, "train/global_grad_norm": 0.20814913511276245}

metrics/jsonlines/train_data_info.jsonl

@@ -0,0 +1 @@
+{"step": 0, "train_data_info/vocab_size": 50277, "train_data_info/global_tokens_per_batch": 2097152, "train_data_info/local_tokens_per_batch": 2097152, "train_data_info/batch_len": 2048, "train_data_info/seq_len": 2048, "train_data_info/total_tokens": 2055208960, "train_data_info/global_batch_size": 1024, "train_data_info/local_batch_size": 1024}

metrics/jsonlines/train_eval.jsonl

@@ -0,0 +1,19 @@
+{"step": 104857600, "train_eval/train_token_count": 104857600, "train_eval/train_batch_count": 50, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 465.39904637495056, "train_eval/train_update_time": 282.29461103421636, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 8.262738060667653, "train_eval/perplexity_len_2048": 3876.6942059185167, "train_eval/loss_avg_len_1024": 8.26358599447005, "train_eval/perplexity_len_1024": 3879.9827800268095, "train_eval/loss_avg_len_512": 8.264393082915195, "train_eval/perplexity_len_512": 3883.115533330371}
+{"step": 209715200, "train_eval/train_token_count": 209715200, "train_eval/train_batch_count": 100, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 926.8863875919487, "train_eval/train_update_time": 560.9345708230976, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 6.3983804440147285, "train_eval/perplexity_len_2048": 600.8711050210618, "train_eval/loss_avg_len_1024": 6.4026496673026125, "train_eval/perplexity_len_1024": 603.4418415551971, "train_eval/loss_avg_len_512": 6.409003813466989, "train_eval/perplexity_len_512": 607.2884070958403}
+{"step": 314572800, "train_eval/train_token_count": 314572800, "train_eval/train_batch_count": 150, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 1478.8752941149287, "train_eval/train_update_time": 839.5983157731825, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 5.694461489945333, "train_eval/perplexity_len_2048": 297.21669630360435, "train_eval/loss_avg_len_1024": 5.700237832467756, "train_eval/perplexity_len_1024": 298.9384897918442, "train_eval/loss_avg_len_512": 5.711870585765283, "train_eval/perplexity_len_512": 302.4362724727184}
+{"step": 419430400, "train_eval/train_token_count": 419430400, "train_eval/train_batch_count": 200, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 1939.2464721049182, "train_eval/train_update_time": 1118.2928310850402, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 5.298926106473736, "train_eval/perplexity_len_2048": 200.12178504909002, "train_eval/loss_avg_len_1024": 5.307277373590877, "train_eval/perplexity_len_1024": 201.80005361248934, "train_eval/loss_avg_len_512": 5.322380470788557, "train_eval/perplexity_len_512": 204.8709913985368}
+{"step": 524288000, "train_eval/train_token_count": 524288000, "train_eval/train_batch_count": 250, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 2491.0224517869065, "train_eval/train_update_time": 1396.97127348301, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 5.0484253960391285, "train_eval/perplexity_len_2048": 155.77698421365454, "train_eval/loss_avg_len_1024": 5.056525684503576, "train_eval/perplexity_len_1024": 157.04394717703963, "train_eval/loss_avg_len_512": 5.07371008965827, "train_eval/perplexity_len_512": 159.76597522371915}
+{"step": 629145600, "train_eval/train_token_count": 629145600, "train_eval/train_batch_count": 300, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 2951.315687590977, "train_eval/train_update_time": 1675.687991371029, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.881842594247683, "train_eval/perplexity_len_2048": 131.87342938039131, "train_eval/loss_avg_len_1024": 4.889910663574483, "train_eval/perplexity_len_1024": 132.94169698475255, "train_eval/loss_avg_len_512": 4.908835977258713, "train_eval/perplexity_len_512": 135.4816189020749}
+{"step": 734003200, "train_eval/train_token_count": 734003200, "train_eval/train_batch_count": 350, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 3502.8775023539783, "train_eval/train_update_time": 1954.4126074366504, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.751916624681217, "train_eval/perplexity_len_2048": 115.80602865192392, "train_eval/loss_avg_len_1024": 4.763219690434598, "train_eval/perplexity_len_1024": 117.12241740770119, "train_eval/loss_avg_len_512": 4.7850995411058825, "train_eval/perplexity_len_512": 119.71327888296197}
+{"step": 838860800, "train_eval/train_token_count": 838860800, "train_eval/train_batch_count": 400, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 3963.4901722619543, "train_eval/train_update_time": 2233.1504810712067, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.652960418538351, "train_eval/perplexity_len_2048": 104.89505965531511, "train_eval/loss_avg_len_1024": 4.663964104556035, "train_eval/perplexity_len_1024": 106.05566571832371, "train_eval/loss_avg_len_512": 4.687754835745363, "train_eval/perplexity_len_512": 108.60906068541759}
+{"step": 943718400, "train_eval/train_token_count": 943718400, "train_eval/train_batch_count": 450, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 4515.121015235898, "train_eval/train_update_time": 2511.850276188343, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.568486952575722, "train_eval/perplexity_len_2048": 96.39814441032826, "train_eval/loss_avg_len_1024": 4.583207558359009, "train_eval/perplexity_len_1024": 97.82767948912803, "train_eval/loss_avg_len_512": 4.6114602793406085, "train_eval/perplexity_len_512": 100.6309917533228}
+{"step": 1048576000, "train_eval/train_token_count": 1048576000, "train_eval/train_batch_count": 500, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 4975.63103498891, "train_eval/train_update_time": 2790.5577287059277, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.499369324265025, "train_eval/perplexity_len_2048": 89.96037757855736, "train_eval/loss_avg_len_1024": 4.513876748953916, "train_eval/perplexity_len_1024": 91.27498369940146, "train_eval/loss_avg_len_512": 4.5444240906891356, "train_eval/perplexity_len_512": 94.10621495483142}
+{"step": 1153433600, "train_eval/train_token_count": 1153433600, "train_eval/train_batch_count": 550, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 5527.108590850956, "train_eval/train_update_time": 3069.2099333773367, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.433416955778939, "train_eval/perplexity_len_2048": 84.21869738313693, "train_eval/loss_avg_len_1024": 4.447785850001091, "train_eval/perplexity_len_1024": 85.43756284075022, "train_eval/loss_avg_len_512": 4.478819852198358, "train_eval/perplexity_len_512": 88.1306041408694}
+{"step": 1258291200, "train_eval/train_token_count": 1258291200, "train_eval/train_batch_count": 600, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 5987.366702231928, "train_eval/train_update_time": 3347.8448207870824, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.377168502442927, "train_eval/perplexity_len_2048": 79.61229195254344, "train_eval/loss_avg_len_1024": 4.392655870016097, "train_eval/perplexity_len_1024": 80.85487410774638, "train_eval/loss_avg_len_512": 4.42795208584983, "train_eval/perplexity_len_512": 83.75970845603244}
+{"step": 1363148800, "train_eval/train_token_count": 1363148800, "train_eval/train_batch_count": 650, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 6538.855644405005, "train_eval/train_update_time": 3626.554348448757, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.331787606836443, "train_eval/perplexity_len_2048": 76.08016650881756, "train_eval/loss_avg_len_1024": 4.353677473009084, "train_eval/perplexity_len_1024": 77.76391242617439, "train_eval/loss_avg_len_512": 4.3933230580473905, "train_eval/perplexity_len_512": 80.90883751188974}
+{"step": 1468006400, "train_eval/train_token_count": 1468006400, "train_eval/train_batch_count": 700, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 6999.246100232936, "train_eval/train_update_time": 3905.29758988251, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.285216300610045, "train_eval/perplexity_len_2048": 72.61825239341289, "train_eval/loss_avg_len_1024": 4.308363595821029, "train_eval/perplexity_len_1024": 74.31877382787907, "train_eval/loss_avg_len_512": 4.353135664994989, "train_eval/perplexity_len_512": 77.72179072718198}
+{"step": 1572864000, "train_eval/train_token_count": 1572864000, "train_eval/train_batch_count": 750, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 7550.323625019984, "train_eval/train_update_time": 4184.042193662957, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.247887878154143, "train_eval/perplexity_len_2048": 69.95749743590287, "train_eval/loss_avg_len_1024": 4.2724517078520154, "train_eval/perplexity_len_1024": 71.69720090430816, "train_eval/loss_avg_len_512": 4.320460617008357, "train_eval/perplexity_len_512": 75.22326943058576}
+{"step": 1677721600, "train_eval/train_token_count": 1677721600, "train_eval/train_batch_count": 800, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 8010.43765124795, "train_eval/train_update_time": 4462.722712960327, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.220693094336503, "train_eval/perplexity_len_2048": 68.08065425680356, "train_eval/loss_avg_len_1024": 4.248167304437285, "train_eval/perplexity_len_1024": 69.97704813075109, "train_eval/loss_avg_len_512": 4.298565637756401, "train_eval/perplexity_len_512": 73.59415727671168}
+{"step": 1782579200, "train_eval/train_token_count": 1782579200, "train_eval/train_batch_count": 850, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 8561.475588510977, "train_eval/train_update_time": 4741.406085017603, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.193676541369477, "train_eval/perplexity_len_2048": 66.2659732436429, "train_eval/loss_avg_len_1024": 4.216801415602568, "train_eval/perplexity_len_1024": 67.81622110062538, "train_eval/loss_avg_len_512": 4.269361495983503, "train_eval/perplexity_len_512": 71.47598334374155}
+{"step": 1887436800, "train_eval/train_token_count": 1887436800, "train_eval/train_batch_count": 900, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 9022.126939511974, "train_eval/train_update_time": 5020.160163923283, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.187023427749736, "train_eval/perplexity_len_2048": 65.82656154335335, "train_eval/loss_avg_len_1024": 4.213506009909706, "train_eval/perplexity_len_1024": 67.5931069672473, "train_eval/loss_avg_len_512": 4.267473418653644, "train_eval/perplexity_len_512": 71.34115847989767}
+{"step": 1992294400, "train_eval/train_token_count": 1992294400, "train_eval/train_batch_count": 950, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 9573.927157147904, "train_eval/train_update_time": 5298.920672355802, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.17461025829327, "train_eval/perplexity_len_2048": 65.01449586199058, "train_eval/loss_avg_len_1024": 4.2044485659670325, "train_eval/perplexity_len_1024": 66.9836504157636, "train_eval/loss_avg_len_512": 4.257914591990775, "train_eval/perplexity_len_512": 70.6624696143801}

metrics/jsonlines/val.jsonl

@@ -0,0 +1,49 @@
+{"step": 41943040, "val/train_token_count": 41943040, "val/train_batch_count": 20, "val/train_flop_count": 0, "val/train_total_time": 115.38901872490533, "val/train_update_time": 115.08477980294265, "val/loss": 8.017322596772551, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.8957867800491, "val/val_tokens_per_second": 450625.9470432396, "val/loss_avg_len_2048": 8.017322596772551, "val/perplexity_len_2048": 3033.045765536961, "val/loss_avg_len_1024": 8.01611629340169, "val/perplexity_len_1024": 3029.389198114233, "val/loss_avg_len_512": 8.016581488862169, "val/perplexity_len_512": 3030.798784058288}
+{"step": 83886080, "val/train_token_count": 83886080, "val/train_batch_count": 40, "val/train_flop_count": 0, "val/train_total_time": 318.01136298198253, "val/train_update_time": 226.56994250579737, "val/loss": 7.168800498851901, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 91.53357388603035, "val/val_tokens_per_second": 447486.0781793556, "val/loss_avg_len_2048": 7.168800498851901, "val/perplexity_len_2048": 1298.286372446997, "val/loss_avg_len_1024": 7.16922726986399, "val/perplexity_len_1024": 1298.8405616836865, "val/loss_avg_len_512": 7.17253666183427, "val/perplexity_len_512": 1303.1460545614693}
+{"step": 125829120, "val/train_token_count": 125829120, "val/train_batch_count": 60, "val/train_flop_count": 0, "val/train_total_time": 521.2494050179375, "val/train_update_time": 338.0269747101702, "val/loss": 6.681955375412503, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.73258669499774, "val/val_tokens_per_second": 451436.48486170854, "val/loss_avg_len_2048": 6.681955375412503, "val/perplexity_len_2048": 797.8777381810718, "val/loss_avg_len_1024": 6.683395165921888, "val/perplexity_len_1024": 799.0273423721623, "val/loss_avg_len_512": 6.689393609371596, "val/perplexity_len_512": 803.834666516683}
+{"step": 167772160, "val/train_token_count": 167772160, "val/train_batch_count": 80, "val/train_flop_count": 0, "val/train_total_time": 723.6805678269593, "val/train_update_time": 449.4893446461065, "val/loss": 6.252993275996973, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 91.52609027700964, "val/val_tokens_per_second": 447522.66677219473, "val/loss_avg_len_2048": 6.252993275996973, "val/perplexity_len_2048": 519.5657029597602, "val/loss_avg_len_1024": 6.255989201308694, "val/perplexity_len_1024": 521.124617029408, "val/loss_avg_len_512": 6.264971070884261, "val/perplexity_len_512": 525.82637404726}
+{"step": 209715200, "val/train_token_count": 209715200, "val/train_batch_count": 100, "val/train_flop_count": 0, "val/train_total_time": 926.8863875919487, "val/train_update_time": 560.9345708230976, "val/loss": 5.955217036969051, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.68943495198619, "val/val_tokens_per_second": 451651.2868526031, "val/loss_avg_len_2048": 5.955217036969051, "val/perplexity_len_2048": 385.76062591006854, "val/loss_avg_len_1024": 5.959280439781304, "val/perplexity_len_1024": 387.3313157338021, "val/loss_avg_len_512": 5.970304897817318, "val/perplexity_len_512": 391.6250581739251}
+{"step": 251658240, "val/train_token_count": 251658240, "val/train_batch_count": 120, "val/train_flop_count": 0, "val/train_total_time": 1129.7145008929074, "val/train_update_time": 672.3896088181064, "val/loss": 5.727587048024056, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.61158224998508, "val/val_tokens_per_second": 452039.34180287144, "val/loss_avg_len_2048": 5.727587048024056, "val/perplexity_len_2048": 307.227049152543, "val/loss_avg_len_1024": 5.732679790063948, "val/perplexity_len_1024": 308.79566815721904, "val/loss_avg_len_512": 5.74529507358959, "val/perplexity_len_512": 312.7158884172708}
+{"step": 293601280, "val/train_token_count": 293601280, "val/train_batch_count": 140, "val/train_flop_count": 0, "val/train_total_time": 1332.0184532779967, "val/train_update_time": 783.8552743501496, "val/loss": 5.5462320905009985, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.99034959904384, "val/val_tokens_per_second": 450157.6285891138, "val/loss_avg_len_2048": 5.5462320905009985, "val/perplexity_len_2048": 256.2701318031649, "val/loss_avg_len_1024": 5.552394144184422, "val/perplexity_len_1024": 257.8541575263273, "val/loss_avg_len_512": 5.566559319449496, "val/perplexity_len_512": 261.5326989437206}
+{"step": 335544320, "val/train_token_count": 335544320, "val/train_batch_count": 160, "val/train_flop_count": 0, "val/train_total_time": 1534.7253116948996, "val/train_update_time": 895.3389847053913, "val/loss": 5.397562808819162, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.53677860402968, "val/val_tokens_per_second": 452412.82748905895, "val/loss_avg_len_2048": 5.397562808819162, "val/perplexity_len_2048": 220.86746347143392, "val/loss_avg_len_1024": 5.404754845878948, "val/perplexity_len_1024": 222.4616764014897, "val/loss_avg_len_512": 5.420072977858782, "val/perplexity_len_512": 225.8956072782415}
+{"step": 377487360, "val/train_token_count": 377487360, "val/train_batch_count": 180, "val/train_flop_count": 0, "val/train_total_time": 1736.965916060959, "val/train_update_time": 1006.818993799272, "val/loss": 5.266776608066797, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.56331127206795, "val/val_tokens_per_second": 452280.28243080724, "val/loss_avg_len_2048": 5.266776608066797, "val/perplexity_len_2048": 193.79029253177575, "val/loss_avg_len_1024": 5.274747739335196, "val/perplexity_len_1024": 195.34119339740653, "val/loss_avg_len_512": 5.2913418367584235, "val/perplexity_len_512": 198.6097485495165}
+{"step": 419430400, "val/train_token_count": 419430400, "val/train_batch_count": 200, "val/train_flop_count": 0, "val/train_total_time": 1939.2464721049182, "val/train_update_time": 1118.2928310850402, "val/loss": 5.159342473128369, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.51000242400914, "val/val_tokens_per_second": 452546.6678049137, "val/loss_avg_len_2048": 5.159342473128369, "val/perplexity_len_2048": 174.04997543648554, "val/loss_avg_len_1024": 5.168076667023404, "val/perplexity_len_1024": 175.57681984049975, "val/loss_avg_len_512": 5.186009379204084, "val/perplexity_len_512": 178.75378911651651}
+{"step": 461373440, "val/train_token_count": 461373440, "val/train_batch_count": 220, "val/train_flop_count": 0, "val/train_total_time": 2141.922117186943, "val/train_update_time": 1229.7717256471515, "val/loss": 5.0615160507458965, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.53271648404188, "val/val_tokens_per_second": 452433.12683785404, "val/loss_avg_len_2048": 5.0615160507458965, "val/perplexity_len_2048": 157.82961273853934, "val/loss_avg_len_1024": 5.071083297332656, "val/perplexity_len_1024": 159.34685389944232, "val/loss_avg_len_512": 5.090461814133264, "val/perplexity_len_512": 162.46487330617947}
+{"step": 503316480, "val/train_token_count": 503316480, "val/train_batch_count": 240, "val/train_flop_count": 0, "val/train_total_time": 2344.147989144898, "val/train_update_time": 1341.2354510270525, "val/loss": 4.984231026970013, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 91.01652396703139, "val/val_tokens_per_second": 450028.1730692858, "val/loss_avg_len_2048": 4.984231026970013, "val/perplexity_len_2048": 146.09119163551048, "val/loss_avg_len_1024": 4.994521487932326, "val/perplexity_len_1024": 147.60229900095914, "val/loss_avg_len_512": 5.01494721182771, "val/perplexity_len_512": 150.64818411371755}
+{"step": 545259520, "val/train_token_count": 545259520, "val/train_batch_count": 260, "val/train_flop_count": 0, "val/train_total_time": 2546.883552026935, "val/train_update_time": 1452.7241989910835, "val/loss": 4.916793563892343, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.51194408303127, "val/val_tokens_per_second": 452536.9597898072, "val/loss_avg_len_2048": 4.916793563892343, "val/perplexity_len_2048": 136.56402658642736, "val/loss_avg_len_1024": 4.927482071626792, "val/perplexity_len_1024": 138.03152093214322, "val/loss_avg_len_512": 4.948590329375863, "val/perplexity_len_512": 140.97609392550117}
+{"step": 587202560, "val/train_token_count": 587202560, "val/train_batch_count": 280, "val/train_flop_count": 0, "val/train_total_time": 2749.1005825489992, "val/train_update_time": 1564.2029820160242, "val/loss": 4.860561983227101, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.51377508195583, "val/val_tokens_per_second": 452527.805440804, "val/loss_avg_len_2048": 4.860561983227101, "val/perplexity_len_2048": 129.09673192321375, "val/loss_avg_len_1024": 4.871906842135312, "val/perplexity_len_1024": 130.5696553890599, "val/loss_avg_len_512": 4.894014172679093, "val/perplexity_len_512": 133.4883452697447}
+{"step": 629145600, "val/train_token_count": 629145600, "val/train_batch_count": 300, "val/train_flop_count": 0, "val/train_total_time": 2951.315687590977, "val/train_update_time": 1675.687991371029, "val/loss": 4.811751918288158, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.51711130701005, "val/val_tokens_per_second": 452511.12644408783, "val/loss_avg_len_2048": 4.811751918288158, "val/perplexity_len_2048": 122.94682173759111, "val/loss_avg_len_1024": 4.823616650856426, "val/perplexity_len_1024": 124.4142409516568, "val/loss_avg_len_512": 4.846837456004229, "val/perplexity_len_512": 127.33704337975502}
+{"step": 671088640, "val/train_token_count": 671088640, "val/train_batch_count": 320, "val/train_flop_count": 0, "val/train_total_time": 3153.9731954459567, "val/train_update_time": 1787.1771466861246, "val/loss": 4.759777876929636, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.7631790220039, "val/val_tokens_per_second": 451284.32522256614, "val/loss_avg_len_2048": 4.759777876929636, "val/perplexity_len_2048": 116.71999681530721, "val/loss_avg_len_1024": 4.771922723843483, "val/perplexity_len_1024": 118.14618620443298, "val/loss_avg_len_512": 4.795630532579589, "val/perplexity_len_512": 120.98063997389404}
+{"step": 713031680, "val/train_token_count": 713031680, "val/train_batch_count": 340, "val/train_flop_count": 0, "val/train_total_time": 3356.450910591986, "val/train_update_time": 1898.6706604874926, "val/loss": 4.716705958361551, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.57427461701445, "val/val_tokens_per_second": 452225.53725321946, "val/loss_avg_len_2048": 4.716705958361551, "val/perplexity_len_2048": 111.79937365796825, "val/loss_avg_len_1024": 4.729562304343749, "val/perplexity_len_1024": 113.24598422400375, "val/loss_avg_len_512": 4.754220814034343, "val/perplexity_len_512": 116.07317533024096}
+{"step": 754974720, "val/train_token_count": 754974720, "val/train_batch_count": 360, "val/train_flop_count": 0, "val/train_total_time": 3558.7265775619308, "val/train_update_time": 2010.1547237539198, "val/loss": 4.67779298429757, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.67017155699432, "val/val_tokens_per_second": 451747.24274402607, "val/loss_avg_len_2048": 4.67779298429757, "val/perplexity_len_2048": 107.53248460560826, "val/loss_avg_len_1024": 4.691213681914146, "val/perplexity_len_1024": 108.98537314715132, "val/loss_avg_len_512": 4.716703379452508, "val/perplexity_len_512": 111.79908533792432}
+{"step": 796917760, "val/train_token_count": 796917760, "val/train_batch_count": 380, "val/train_flop_count": 0, "val/train_total_time": 3761.109741097898, "val/train_update_time": 2121.6608164007775, "val/loss": 4.640750338151609, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.67889700899832, "val/val_tokens_per_second": 451703.7739876283, "val/loss_avg_len_2048": 4.640750338151609, "val/perplexity_len_2048": 103.62207001306183, "val/loss_avg_len_1024": 4.654702061567177, "val/perplexity_len_1024": 105.07790858728595, "val/loss_avg_len_512": 4.681117743289098, "val/perplexity_len_512": 107.89059919314577}
+{"step": 838860800, "val/train_token_count": 838860800, "val/train_batch_count": 400, "val/train_flop_count": 0, "val/train_total_time": 3963.4901722619543, "val/train_update_time": 2233.1504810712067, "val/loss": 4.607613671057416, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.60668137599714, "val/val_tokens_per_second": 452063.79240428534, "val/loss_avg_len_2048": 4.607613671057416, "val/perplexity_len_2048": 100.24464728119722, "val/loss_avg_len_1024": 4.6223377861871855, "val/perplexity_len_1024": 101.73158103702862, "val/loss_avg_len_512": 4.650039090201073, "val/perplexity_len_512": 104.58907390513586}
+{"step": 880803840, "val/train_token_count": 880803840, "val/train_batch_count": 420, "val/train_flop_count": 0, "val/train_total_time": 4166.242571576964, "val/train_update_time": 2344.6231456701644, "val/loss": 4.57813637526834, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.58338582294527, "val/val_tokens_per_second": 452180.0507662698, "val/loss_avg_len_2048": 4.57813637526834, "val/perplexity_len_2048": 97.33283320365324, "val/loss_avg_len_1024": 4.593541098429355, "val/perplexity_len_1024": 98.84382689289926, "val/loss_avg_len_512": 4.622042558031157, "val/perplexity_len_512": 101.70155144295813}
+{"step": 922746880, "val/train_token_count": 922746880, "val/train_batch_count": 440, "val/train_flop_count": 0, "val/train_total_time": 4368.526084997924, "val/train_update_time": 2456.1160258102464, "val/loss": 4.545171438023285, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.74309363192879, "val/val_tokens_per_second": 451384.2140553587, "val/loss_avg_len_2048": 4.545171438023285, "val/perplexity_len_2048": 94.17657127073271, "val/loss_avg_len_1024": 4.560983082159796, "val/perplexity_len_1024": 95.67749244735029, "val/loss_avg_len_512": 4.5904487324811525, "val/perplexity_len_512": 98.53863772991662}
+{"step": 964689920, "val/train_token_count": 964689920, "val/train_batch_count": 460, "val/train_flop_count": 0, "val/train_total_time": 4570.982028090977, "val/train_update_time": 2567.596293253242, "val/loss": 4.517566748579941, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.59338419197593, "val/val_tokens_per_second": 452130.14576430764, "val/loss_avg_len_2048": 4.517566748579941, "val/perplexity_len_2048": 91.61241052467754, "val/loss_avg_len_1024": 4.5339546799753325, "val/perplexity_len_1024": 93.12611781174976, "val/loss_avg_len_512": 4.564446135131549, "val/perplexity_len_512": 96.00940305219359}
+{"step": 1006632960, "val/train_token_count": 1006632960, "val/train_batch_count": 480, "val/train_flop_count": 0, "val/train_total_time": 4773.292612125981, "val/train_update_time": 2679.07335356588, "val/loss": 4.490669789551548, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.61795308801811, "val/val_tokens_per_second": 452007.5614620775, "val/loss_avg_len_2048": 4.490669789551548, "val/perplexity_len_2048": 89.18115848713532, "val/loss_avg_len_1024": 4.507707643355197, "val/perplexity_len_1024": 90.71363198575763, "val/loss_avg_len_512": 4.539340242150892, "val/perplexity_len_512": 93.62900726510776}
+{"step": 1048576000, "val/train_token_count": 1048576000, "val/train_batch_count": 500, "val/train_flop_count": 0, "val/train_total_time": 4975.63103498891, "val/train_update_time": 2790.5577287059277, "val/loss": 4.4664931430107915, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.63011032703798, "val/val_tokens_per_second": 451946.92858914315, "val/loss_avg_len_2048": 4.4664931430107915, "val/perplexity_len_2048": 87.05091200992092, "val/loss_avg_len_1024": 4.484059858988878, "val/perplexity_len_1024": 88.59362115171373, "val/loss_avg_len_512": 4.5166467366272585, "val/perplexity_len_512": 91.52816477147304}
+{"step": 1090519040, "val/train_token_count": 1090519040, "val/train_batch_count": 520, "val/train_flop_count": 0, "val/train_total_time": 5178.418338532909, "val/train_update_time": 2902.025444978848, "val/loss": 4.441390738584008, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.61769351002295, "val/val_tokens_per_second": 452008.8562557547, "val/loss_avg_len_2048": 4.441390738584008, "val/perplexity_len_2048": 84.8929234780783, "val/loss_avg_len_1024": 4.459845923438808, "val/perplexity_len_1024": 86.47418442487788, "val/loss_avg_len_512": 4.493771929252893, "val/perplexity_len_512": 89.45824045076729}
+{"step": 1132462080, "val/train_token_count": 1132462080, "val/train_batch_count": 540, "val/train_flop_count": 0, "val/train_total_time": 5380.720167659922, "val/train_update_time": 3013.4729040842503, "val/loss": 4.420545830946578, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.53602080291603, "val/val_tokens_per_second": 452416.6142574795, "val/loss_avg_len_2048": 4.420545830946578, "val/perplexity_len_2048": 83.14165426318776, "val/loss_avg_len_1024": 4.439822294117836, "val/perplexity_len_1024": 84.75987800652325, "val/loss_avg_len_512": 4.474853580821212, "val/perplexity_len_512": 87.78174653751046}
+{"step": 1174405120, "val/train_token_count": 1174405120, "val/train_batch_count": 560, "val/train_flop_count": 0, "val/train_total_time": 5582.958667394938, "val/train_update_time": 3124.937779762433, "val/loss": 4.39603335383758, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.5017854050966, "val/val_tokens_per_second": 452587.75632611266, "val/loss_avg_len_2048": 4.39603335383758, "val/perplexity_len_2048": 81.12842182839486, "val/loss_avg_len_1024": 4.415969875032036, "val/perplexity_len_1024": 82.762070859431, "val/loss_avg_len_512": 4.452327949493192, "val/perplexity_len_512": 85.82651140438335}
+{"step": 1216348160, "val/train_token_count": 1216348160, "val/train_batch_count": 580, "val/train_flop_count": 0, "val/train_total_time": 5785.135669226991, "val/train_update_time": 3236.387389887357, "val/loss": 4.3746475970230305, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.56642184499651, "val/val_tokens_per_second": 452264.74851907714, "val/loss_avg_len_2048": 4.3746475970230305, "val/perplexity_len_2048": 79.41184964846552, "val/loss_avg_len_1024": 4.395640490190779, "val/perplexity_len_1024": 81.09655568067153, "val/loss_avg_len_512": 4.4339910840201195, "val/perplexity_len_512": 84.26706359861659}
+{"step": 1258291200, "val/train_token_count": 1258291200, "val/train_batch_count": 600, "val/train_flop_count": 0, "val/train_total_time": 5987.366702231928, "val/train_update_time": 3347.8448207870824, "val/loss": 4.355698767514504, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.80764411704149, "val/val_tokens_per_second": 451063.34822657524, "val/loss_avg_len_2048": 4.355698767514504, "val/perplexity_len_2048": 77.9212551595122, "val/loss_avg_len_1024": 4.377580918695685, "val/perplexity_len_1024": 79.64513212710933, "val/loss_avg_len_512": 4.417377031392325, "val/perplexity_len_512": 82.87861201041235}
+{"step": 1300234240, "val/train_token_count": 1300234240, "val/train_batch_count": 620, "val/train_flop_count": 0, "val/train_total_time": 6190.319736288977, "val/train_update_time": 3459.312256404897, "val/loss": 4.333859468727117, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.45447246590629, "val/val_tokens_per_second": 452824.4859914303, "val/loss_avg_len_2048": 4.333859468727117, "val/perplexity_len_2048": 76.23795751058033, "val/loss_avg_len_1024": 4.35671562793199, "val/perplexity_len_1024": 78.00053049871111, "val/loss_avg_len_512": 4.398111169292033, "val/perplexity_len_512": 81.2971669669832}
+{"step": 1342177280, "val/train_token_count": 1342177280, "val/train_batch_count": 640, "val/train_flop_count": 0, "val/train_total_time": 6392.506873650011, "val/train_update_time": 3570.8129316339036, "val/loss": 4.315572682284401, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.48178494896274, "val/val_tokens_per_second": 452687.7981364309, "val/loss_avg_len_2048": 4.315572682284401, "val/perplexity_len_2048": 74.85648015118835, "val/loss_avg_len_1024": 4.339295653387905, "val/perplexity_len_1024": 76.65352966583826, "val/loss_avg_len_512": 4.382252874559444, "val/perplexity_len_512": 80.01810123837882}
+{"step": 1384120320, "val/train_token_count": 1384120320, "val/train_batch_count": 660, "val/train_flop_count": 0, "val/train_total_time": 6594.719191052951, "val/train_update_time": 3682.304503756459, "val/loss": 4.299363855137489, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.58356982597616, "val/val_tokens_per_second": 452179.1322498103, "val/loss_avg_len_2048": 4.299363855137489, "val/perplexity_len_2048": 73.65292486373085, "val/loss_avg_len_1024": 4.323835089938761, "val/perplexity_len_1024": 75.47753708539467, "val/loss_avg_len_512": 4.368103061350993, "val/perplexity_len_512": 78.89383290411739}
+{"step": 1426063360, "val/train_token_count": 1426063360, "val/train_batch_count": 680, "val/train_flop_count": 0, "val/train_total_time": 6797.037191045936, "val/train_update_time": 3793.7997314956738, "val/loss": 4.2817527375663165, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.48214890307281, "val/val_tokens_per_second": 452685.97725146404, "val/loss_avg_len_2048": 4.2817527375663165, "val/perplexity_len_2048": 72.36716956972212, "val/loss_avg_len_1024": 4.30717062690584, "val/perplexity_len_1024": 74.23016670416823, "val/loss_avg_len_512": 4.353055857041944, "val/perplexity_len_512": 77.71558815766758}
+{"step": 1468006400, "val/train_token_count": 1468006400, "val/train_batch_count": 700, "val/train_flop_count": 0, "val/train_total_time": 6999.246100232936, "val/train_update_time": 3905.29758988251, "val/loss": 4.266338244345109, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.41607927205041, "val/val_tokens_per_second": 453016.76792196004, "val/loss_avg_len_2048": 4.266338244345109, "val/perplexity_len_2048": 71.26021977810917, "val/loss_avg_len_1024": 4.292485382087575, "val/perplexity_len_1024": 73.14804360172089, "val/loss_avg_len_512": 4.339678274843562, "val/perplexity_len_512": 76.68286456266289}
+{"step": 1509949440, "val/train_token_count": 1509949440, "val/train_batch_count": 720, "val/train_flop_count": 0, "val/train_total_time": 7201.8562647239305, "val/train_update_time": 4016.812845747103, "val/loss": 4.252442864475725, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.41256658593193, "val/val_tokens_per_second": 453034.36841459293, "val/loss_avg_len_2048": 4.252442864475725, "val/perplexity_len_2048": 70.27687971857087, "val/loss_avg_len_1024": 4.279593376897369, "val/perplexity_len_1024": 72.2110713468667, "val/loss_avg_len_512": 4.328346583297849, "val/perplexity_len_512": 75.81882276789578}
+{"step": 1551892480, "val/train_token_count": 1551892480, "val/train_batch_count": 740, "val/train_flop_count": 0, "val/train_total_time": 7404.001054160995, "val/train_update_time": 4128.298564241966, "val/loss": 4.239966365041072, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.46125932002906, "val/val_tokens_per_second": 452790.512843668, "val/loss_avg_len_2048": 4.239966365041072, "val/perplexity_len_2048": 69.40551734776977, "val/loss_avg_len_1024": 4.2677443491777405, "val/perplexity_len_1024": 71.36048959593073, "val/loss_avg_len_512": 4.317693537768815, "val/perplexity_len_512": 75.015408400013}
+{"step": 1593835520, "val/train_token_count": 1593835520, "val/train_batch_count": 760, "val/train_flop_count": 0, "val/train_total_time": 7606.181935134926, "val/train_update_time": 4239.792019490269, "val/loss": 4.228949285643083, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.43556012096815, "val/val_tokens_per_second": 452919.18295426277, "val/loss_avg_len_2048": 4.228949285643083, "val/perplexity_len_2048": 68.64506791008688, "val/loss_avg_len_1024": 4.257730329607474, "val/perplexity_len_1024": 70.64945037883355, "val/loss_avg_len_512": 4.309273659745045, "val/perplexity_len_512": 74.38643944816678}
+{"step": 1635778560, "val/train_token_count": 1635778560, "val/train_batch_count": 780, "val/train_flop_count": 0, "val/train_total_time": 7808.319425186957, "val/train_update_time": 4351.274368522107, "val/loss": 4.218897107815556, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.45598102302756, "val/val_tokens_per_second": 452816.93412371184, "val/loss_avg_len_2048": 4.218897107815556, "val/perplexity_len_2048": 67.95849205309356, "val/loss_avg_len_1024": 4.248032534746873, "val/perplexity_len_1024": 69.96761798110202, "val/loss_avg_len_512": 4.300288976043649, "val/perplexity_len_512": 73.72109425192328}
+{"step": 1677721600, "val/train_token_count": 1677721600, "val/train_batch_count": 800, "val/train_flop_count": 0, "val/train_total_time": 8010.43765124795, "val/train_update_time": 4462.722712960327, "val/loss": 4.210539670823771, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.42771953193005, "val/val_tokens_per_second": 452958.453580564, "val/loss_avg_len_2048": 4.210539670823771, "val/perplexity_len_2048": 67.39289997982114, "val/loss_avg_len_1024": 4.2402994022696285, "val/perplexity_len_1024": 69.42863581835583, "val/loss_avg_len_512": 4.293656023380068, "val/perplexity_len_512": 73.23372386266715}
+{"step": 1719664640, "val/train_token_count": 1719664640, "val/train_batch_count": 820, "val/train_flop_count": 0, "val/train_total_time": 8212.996373801958, "val/train_update_time": 4574.1805559834465, "val/loss": 4.2029221291968835, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.4471801869804, "val/val_tokens_per_second": 452860.99484056735, "val/loss_avg_len_2048": 4.2029221291968835, "val/perplexity_len_2048": 66.88148210534082, "val/loss_avg_len_1024": 4.232967852277168, "val/perplexity_len_1024": 68.92147770319531, "val/loss_avg_len_512": 4.28687209741557, "val/perplexity_len_512": 72.7385930661049}
+{"step": 1761607680, "val/train_token_count": 1761607680, "val/train_batch_count": 840, "val/train_flop_count": 0, "val/train_total_time": 8415.15657795791, "val/train_update_time": 4685.668182700407, "val/loss": 4.196753646510444, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.46505224099383, "val/val_tokens_per_second": 452771.5287322762, "val/loss_avg_len_2048": 4.196753646510444, "val/perplexity_len_2048": 66.4701946548187, "val/loss_avg_len_1024": 4.227167845540075, "val/perplexity_len_1024": 68.5228896921381, "val/loss_avg_len_512": 4.281677430549357, "val/perplexity_len_512": 72.36172001925331}
+{"step": 1803550720, "val/train_token_count": 1803550720, "val/train_batch_count": 860, "val/train_flop_count": 0, "val/train_total_time": 8617.329412178951, "val/train_update_time": 4797.150553135551, "val/loss": 4.191815680753463, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.45923761592712, "val/val_tokens_per_second": 452800.63241200906, "val/loss_avg_len_2048": 4.191815680753463, "val/perplexity_len_2048": 66.1427761656977, "val/loss_avg_len_1024": 4.222529412544612, "val/perplexity_len_1024": 68.20578685830964, "val/loss_avg_len_512": 4.277603877550364, "val/perplexity_len_512": 72.06755028244689}
+{"step": 1845493760, "val/train_token_count": 1845493760, "val/train_batch_count": 880, "val/train_flop_count": 0, "val/train_total_time": 8819.515494428924, "val/train_update_time": 4908.649304769933, "val/loss": 4.187887623292348, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.85943013709038, "val/val_tokens_per_second": 450806.2612565234, "val/loss_avg_len_2048": 4.187887623292348, "val/perplexity_len_2048": 65.88347315226201, "val/loss_avg_len_1024": 4.218744200249576, "val/perplexity_len_1024": 67.94810147990495, "val/loss_avg_len_512": 4.274144274820015, "val/perplexity_len_512": 71.81865597468175}
+{"step": 1887436800, "val/train_token_count": 1887436800, "val/train_batch_count": 900, "val/train_flop_count": 0, "val/train_total_time": 9022.126939511974, "val/train_update_time": 5020.160163923283, "val/loss": 4.1849501522684704, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.70509503502399, "val/val_tokens_per_second": 451573.3100128951, "val/loss_avg_len_2048": 4.1849501522684704, "val/perplexity_len_2048": 65.69022632635689, "val/loss_avg_len_1024": 4.216048343973933, "val/perplexity_len_1024": 67.7651698535888, "val/loss_avg_len_512": 4.271848099075444, "val/perplexity_len_512": 71.6539369031657}
+{"step": 1929379840, "val/train_token_count": 1929379840, "val/train_batch_count": 920, "val/train_flop_count": 0, "val/train_total_time": 9224.990330599947, "val/train_update_time": 5131.661126127117, "val/loss": 4.182838360282803, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.65441882296, "val/val_tokens_per_second": 451825.74144555745, "val/loss_avg_len_2048": 4.182838360282803, "val/perplexity_len_2048": 65.55164860802255, "val/loss_avg_len_1024": 4.214113749123505, "val/perplexity_len_1024": 67.63419843411694, "val/loss_avg_len_512": 4.270228833553196, "val/perplexity_len_512": 71.53800404197908}
+{"step": 1971322880, "val/train_token_count": 1971322880, "val/train_batch_count": 940, "val/train_flop_count": 0, "val/train_total_time": 9427.379897051957, "val/train_update_time": 5243.171735763899, "val/loss": 4.181507043302083, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.67323047306854, "val/val_tokens_per_second": 451732.00277854665, "val/loss_avg_len_2048": 4.181507043302083, "val/perplexity_len_2048": 65.46443665137711, "val/loss_avg_len_1024": 4.212834281807114, "val/perplexity_len_1024": 67.54771802397694, "val/loss_avg_len_512": 4.269042033359408, "val/perplexity_len_512": 71.45315308544407}
+{"step": 2013265920, "val/train_token_count": 2013265920, "val/train_batch_count": 960, "val/train_flop_count": 0, "val/train_total_time": 9629.798734208918, "val/train_update_time": 5354.673169916612, "val/loss": 4.1807237462811635, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.82715408504009, "val/val_tokens_per_second": 450966.45835285966, "val/loss_avg_len_2048": 4.1807237462811635, "val/perplexity_len_2048": 65.4131786309201, "val/loss_avg_len_1024": 4.212053486323543, "val/perplexity_len_1024": 67.49499765540955, "val/loss_avg_len_512": 4.268303510677349, "val/perplexity_len_512": 71.40040279222738}
+{"step": 2055208960, "val/train_token_count": 2055208960, "val/train_batch_count": 980, "val/train_flop_count": 0, "val/train_total_time": 9832.331430819933, "val/train_update_time": 5466.160342909512, "val/loss": 4.180392267811508, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 90.65315544593614, "val/val_tokens_per_second": 451832.0382617877, "val/loss_avg_len_2048": 4.180392267811508, "val/perplexity_len_2048": 65.39149916390902, "val/loss_avg_len_1024": 4.211774798628548, "val/perplexity_len_1024": 67.47619025090711, "val/loss_avg_len_512": 4.26806066927705, "val/perplexity_len_512": 71.38306592357135}

metrics/jsonlines/val_data_info.jsonl

@@ -0,0 +1 @@
+{"step": 0, "val_data_info/vocab_size": 50277, "val_data_info/global_tokens_per_batch": 2048, "val_data_info/local_tokens_per_batch": 2048, "val_data_info/batch_len": 2048, "val_data_info/seq_len": 2048, "val_data_info/total_tokens": 2147483648, "val_data_info/global_batch_size": 1, "val_data_info/local_batch_size": 1}

metrics/npz/train_eval/step-000000104857600.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5edbc3e504381f9fc3799b6feeff24af725eb43cc535c19ea5f9d9c54b3ac325
+size 20540

metrics/npz/train_eval/step-000000209715200.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73d9de1b64eb9c0715c80d1997ae4bd0e5e8c7af00a76ff6215ba7ff8bf5ac66
+size 20540