add remote code + model files

.hydra/config.yaml

@@ -0,0 +1,93 @@
+model:
+  _target_: forgetting_transformer.model.alibi.modeling_alibi.AlibiForCausalLM
+  config:
+    _target_: forgetting_transformer.model.alibi.configuration_alibi.AlibiConfig
+    vocab_size: ???
+    hidden_size: 256
+    hidden_ratio: 4
+    intermediate_size: null
+    num_hidden_layers: 3
+    num_heads: 4
+    num_kv_heads: null
+    hidden_act: swish
+    window_size: null
+    max_position_embeddings: null
+    initializer_range: 0.02
+    elementwise_affine: true
+    norm_eps: 1.0e-06
+    use_cache: true
+    pad_token_id: null
+    bos_token_id: null
+    eos_token_id: null
+    tie_word_embeddings: false
+    attention_bias: false
+    fuse_norm: true
+    fuse_cross_entropy: true
+    use_rope: false
+    use_alibi: true
+optimizer:
+  _target_: torch.optim.AdamW
+  lr: 0.001
+  betas:
+  - 0.9
+  - 0.95
+  weight_decay: 0.1
+schedule:
+  _target_: forgetting_transformer.schedule.warmup_cosine_decay_schedule
+  init_value: 0.0
+  peak_value: ${optimizer.lr}
+  warmup_steps: 20971520
+  decay_steps: ${train.max_tokens}
+  end_value: 0.0
+datamodule:
+  _target_: forgetting_transformer.datamodule.npy.NpyDataModule
+  data_path: ${data_dir}
+  rank: ???
+  world_size: ???
+  train_batch_len: 2048
+  train_batch_size: 1024
+  train_num_workers: 0
+  eval_tokens: 2147483648
+  eval_batch_len: 2048
+  eval_local_batch_size: 1
+  eval_num_workers: 0
+strategy:
+  _target_: lightning.fabric.strategies.FSDPStrategy
+  state_dict_type: full
+  sharding_strategy: FULL_SHARD
+  cpu_offload: false
+exp: alibi_3_4_256
+tag: alibi_3_4_256
+seed: 42
+hf_load_dir: null
+hf_save_dir: null
+hf_load_step: null
+output_dir: ./alibi_3_4_256/
+data_dir: data
+resume: false
+fork_dir: null
+fork_step: null
+log_interval: 20971520
+eval_interval: 41943040
+final_eval: true
+skip_eval: false
+checkpoint_interval: 209715200
+train_eval_interval: 104857600
+checkpoint_keep_interval: 209715200
+fabric:
+  devices: 1
+  precision: 16-mixed
+train:
+  max_tokens: 2097152000
+  grad_acc_tokens: 32768
+  max_grad_norm: 1.0
+  gradient_checkpointing: true
+  bias_weight_decay: false
+  normalization_weight_decay: false
+  conv_weight_decay: true
+eval:
+  min_val_length: 512
+wandb:
+  project: forgetting-transformer
+  mode: online
+  log_dir: ./output/wandb

.hydra/hydra.yaml

@@ -0,0 +1,140 @@
+hydra:
+  run:
+    dir: ${output_dir}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    root: null
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    root: null
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - +experiment/pile/alibi=alibi_3_4_256
+    - fabric.devices=1
+    - fabric.precision=16-mixed
+    - seed=42
+    - exp=alibi_3_4_256
+    - tag=alibi_3_4_256
+    - output_dir=./alibi_3_4_256/
+    - wandb.log_dir=./output/wandb
+    - wandb.mode=online
+    - resume=false
+  job:
+    name: train
+    chdir: null
+    override_dirname: +experiment/pile/alibi=alibi_3_4_256,exp=alibi_3_4_256,fabric.devices=1,fabric.precision=16-mixed,output_dir=./alibi_3_4_256/,resume=false,seed=42,tag=alibi_3_4_256,wandb.log_dir=./output/wandb,wandb.mode=online
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /workspace/forgetting-transformer
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /workspace/forgetting-transformer/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /workspace/forgetting-transformer/alibi_3_4_256
+    choices:
+      experiment/pile/alibi: alibi_3_4_256
+      strategy: fsdp
+      datamodule: npy
+      schedule: warmup_cosine
+      optimizer: adamw
+      model: alibi
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: none
+      hydra/hydra_logging: none
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

.hydra/overrides.yaml

@@ -0,0 +1,10 @@
+- +experiment/pile/alibi=alibi_3_4_256
+- fabric.devices=1
+- fabric.precision=16-mixed
+- seed=42
+- exp=alibi_3_4_256
+- tag=alibi_3_4_256
+- output_dir=./alibi_3_4_256/
+- wandb.log_dir=./output/wandb
+- wandb.mode=online
+- resume=false

__init__.py

@@ -0,0 +1 @@
+# for HF remote code

checkpoints/step-000000209715200.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29fd8196399c4483c8d0dd36172dcd5357b344cb7d987290d3ae3738370af50c
+size 339651594

checkpoints/step-000000419430400.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed03eeedd6740fe5e0b577e6d63b6c367bf73c80639b0bcfbb7333757d85769d
+size 339651594

checkpoints/step-000000629145600.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f5f874678886d8290c617479e7a473d4efdeac4fcaa26d0aed4acbae667faa9
+size 339651594

checkpoints/step-000000838860800.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6b0eef4f33e5c387c29464f25fdd5e054da835fe1a0b100ce87d1bfaabc4af5
+size 339651594

checkpoints/step-000001048576000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97a0fd7c28f539bbc9bfea7c21eee2b0f6929083c5412d9402e58127d1922c7a
+size 339651594

checkpoints/step-000001258291200.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f06d1cbd49c822165da339c488d9cb517b80a6ecddd50208fb6132d298c3e2d9
+size 339651594

checkpoints/step-000001468006400.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7ce6551ef5f5c2f4bea883a8775f2c1822f5a01871fcc647cb49cfb5aa30e5c
+size 339651594

checkpoints/step-000001677721600.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:642ad5472d53df8af94ec16edf351aa79b6eeea3917af3deff0e28b441d7d26a
+size 339651594

checkpoints/step-000001887436800.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c4d01b6fd911272c98ca7d089362fada6cbd2f46789aa50d18feaf23be7ff94
+size 339651594

config.yaml

@@ -0,0 +1,93 @@
+model:
+  _target_: forgetting_transformer.model.alibi.modeling_alibi.AlibiForCausalLM
+  config:
+    _target_: forgetting_transformer.model.alibi.configuration_alibi.AlibiConfig
+    vocab_size: ???
+    hidden_size: 256
+    hidden_ratio: 4
+    intermediate_size: null
+    num_hidden_layers: 3
+    num_heads: 4
+    num_kv_heads: null
+    hidden_act: swish
+    window_size: null
+    max_position_embeddings: null
+    initializer_range: 0.02
+    elementwise_affine: true
+    norm_eps: 1.0e-06
+    use_cache: true
+    pad_token_id: null
+    bos_token_id: null
+    eos_token_id: null
+    tie_word_embeddings: false
+    attention_bias: false
+    fuse_norm: true
+    fuse_cross_entropy: true
+    use_rope: false
+    use_alibi: true
+optimizer:
+  _target_: torch.optim.AdamW
+  lr: 0.001
+  betas:
+  - 0.9
+  - 0.95
+  weight_decay: 0.1
+schedule:
+  _target_: forgetting_transformer.schedule.warmup_cosine_decay_schedule
+  init_value: 0.0
+  peak_value: 0.001
+  warmup_steps: 20971520
+  decay_steps: 2097152000
+  end_value: 0.0
+datamodule:
+  _target_: forgetting_transformer.datamodule.npy.NpyDataModule
+  data_path: /workspace/forgetting-transformer/data
+  rank: ???
+  world_size: ???
+  train_batch_len: 2048
+  train_batch_size: 1024
+  train_num_workers: 0
+  eval_tokens: 2147483648
+  eval_batch_len: 2048
+  eval_local_batch_size: 1
+  eval_num_workers: 0
+strategy:
+  _target_: lightning.fabric.strategies.FSDPStrategy
+  state_dict_type: full
+  sharding_strategy: FULL_SHARD
+  cpu_offload: false
+exp: alibi_3_4_256
+tag: alibi_3_4_256
+seed: 42
+hf_load_dir: null
+hf_save_dir: null
+hf_load_step: null
+output_dir: /workspace/forgetting-transformer/alibi_3_4_256
+data_dir: /workspace/forgetting-transformer/data
+resume: false
+fork_dir: null
+fork_step: null
+log_interval: 20971520
+eval_interval: 41943040
+final_eval: true
+skip_eval: false
+checkpoint_interval: 209715200
+train_eval_interval: 104857600
+checkpoint_keep_interval: 209715200
+fabric:
+  devices: 1
+  precision: 16-mixed
+train:
+  max_tokens: 2097152000
+  grad_acc_tokens: 32768
+  max_grad_norm: 1.0
+  gradient_checkpointing: true
+  bias_weight_decay: false
+  normalization_weight_decay: false
+  conv_weight_decay: true
+eval:
+  min_val_length: 512
+wandb:
+  project: forgetting-transformer
+  mode: online
+  log_dir: ./output/wandb

configuration_transformer.py

@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+
+from typing import Optional
+
+from transformers.configuration_utils import PretrainedConfig
+
+
+class TransformerConfig(PretrainedConfig):
+
+    model_type = 'transformer-project_fox'
+    keys_to_ignore_at_inference = ['past_key_values']
+
+    def __init__(
+        self,
+        vocab_size: int = 32000,
+        hidden_size: int = 2048,
+        hidden_ratio: Optional[int] = 4,
+        intermediate_size: Optional[int] = None,
+        num_hidden_layers: int = 24,
+        num_heads: int = 32,
+        num_kv_heads: int = None,
+        hidden_act: str = "swish",
+        window_size: Optional[int] = None,
+        max_position_embeddings: int = 2048,
+        initializer_range: float = 0.02,
+        elementwise_affine: Optional[bool] = True,
+        norm_eps: float = 1e-6,
+        use_cache: bool = True,
+        pad_token_id: int = None,
+        bos_token_id: int = 1,
+        eos_token_id: int = 2,
+        tie_word_embeddings: bool = False,
+        attention_bias: bool = False,
+        fuse_norm: bool = True,
+        fuse_cross_entropy: bool = True,
+        rope_base: float = 500000.0,
+        use_rope: bool = True,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.hidden_ratio = hidden_ratio
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_heads = num_heads
+        self.num_kv_heads = num_kv_heads
+        self.window_size = window_size
+        self.max_position_embeddings = max_position_embeddings
+
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.elementwise_affine = elementwise_affine
+        self.norm_eps = norm_eps
+        self.use_cache = use_cache
+        self.attention_bias = attention_bias
+        self.fuse_cross_entropy = fuse_cross_entropy
+        self.fuse_norm = fuse_norm
+        self.rope_base = rope_base
+        self.use_rope = use_rope
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )

decay_params.txt

@@ -0,0 +1,20 @@
+_forward_module._fsdp_wrapped_module.model.embeddings.weight
+_forward_module._fsdp_wrapped_module.model.layers.0.attn.q_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.0.attn.k_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.0.attn.v_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.0.attn.o_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.0.mlp.gate_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.0.mlp.down_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.1.attn.q_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.1.attn.k_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.1.attn.v_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.1.attn.o_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.1.mlp.gate_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.1.mlp.down_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.2.attn.q_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.2.attn.k_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.2.attn.v_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.2.attn.o_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.2.mlp.gate_proj.weight
+_forward_module._fsdp_wrapped_module.model.layers.2.mlp.down_proj.weight
+_forward_module._fsdp_wrapped_module.lm_head.weight

logs/2025-10-12_03-08-59.log

@@ -0,0 +1,258 @@
+[2025-10-12 03:08:59][train:372][INFO] All outputs will be saved to `/workspace/forgetting-transformer/alibi_3_4_256`
+[2025-10-12 03:08:59][train:375][INFO] Configuration:
+[2025-10-12 03:08:59][train:380][INFO] Configuration saved to /workspace/forgetting-transformer/alibi_3_4_256/config.yaml.
+[2025-10-12 03:08:59][train:387][INFO] creating datamodule
+[2025-10-12 03:08:59][train:419][INFO] creating model
+[2025-10-12 03:08:59][train:440][INFO] creating optimizer
+[2025-10-12 03:08:59][checkpoint:39][INFO] Not resuming. Deleting existing checkpoints...
+[2025-10-12 03:08:59][logger:256][INFO] Setting up wandb logger...
+[2025-10-12 03:08:59][logger:272][INFO] Not resuming. Creating a new wandb run.
+[2025-10-12 03:09:00][logger:288][INFO] wandb initialized. Run id: y12r42hn
+[2025-10-12 03:09:00][logger:186][INFO] Setting up jsonlines logger...
+[2025-10-12 03:09:00][logger:113][INFO] Setting up npz logger...
+[2025-10-12 03:09:00][logger:171][INFO] [step: 0] [train_data_info/vocab_size: 50277] [train_data_info/global_tokens_per_batch: 2097152] [train_data_info/local_tokens_per_batch: 2097152] [train_data_info/batch_len: 2048] [train_data_info/seq_len: 2048] [train_data_info/total_tokens: 2055208960] [train_data_info/global_batch_size: 1024] [train_data_info/local_batch_size: 1024]
+[2025-10-12 03:09:00][logger:171][INFO] [step: 0] [val_data_info/vocab_size: 50277] [val_data_info/global_tokens_per_batch: 2048] [val_data_info/local_tokens_per_batch: 2048] [val_data_info/batch_len: 2048] [val_data_info/seq_len: 2048] [val_data_info/total_tokens: 2147483648] [val_data_info/global_batch_size: 1] [val_data_info/local_batch_size: 1]
+[2025-10-12 03:09:00][logger:171][INFO] [step: 0] [model_info/total_params: 28299520] [model_info/trainable_params: 28299520] [model_info/embedding_params: 12870912] [model_info/flops_per_token: 0] [model_info/non_embedding_params: 15428608]
+[2025-10-12 03:12:41][utils:57][INFO] [P: 1.00%] [S: 20971520/2097152000] [T: 0:03:39] [ETA: 6:02:05] [loss: 9.772] [tokens/s: 98655.395] [batches/s: 0.047] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 03:15:34][utils:57][INFO] [P: 2.00%] [S: 41943040/2097152000] [T: 0:06:33] [ETA: 5:21:11] [loss: 8.123] [tokens/s: 109115.478] [batches/s: 0.052] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 03:15:34][train:194][INFO] Running validation...
+[2025-10-12 03:20:02][logger:171][INFO] [step: 41943040] [val/train_token_count: 41943040] [val/train_batch_count: 20] [val/train_flop_count: 0] [val/train_total_time: 393.303] [val/train_update_time: 392.719] [val/loss: 8.011] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 267.455] [val/val_tokens_per_second: 153147.316] [val/loss_avg_len_2048: 8.011] [val/perplexity_len_2048: 3014.638] [val/loss_avg_len_1024: 8.010] [val/perplexity_len_1024: 3011.206] [val/loss_avg_len_512: 8.011] [val/perplexity_len_512: 3013.481]
+[2025-10-12 03:23:25][utils:57][INFO] [P: 3.00%] [S: 62914560/2097152000] [T: 0:14:23] [ETA: 7:45:24] [loss: 7.505] [tokens/s: 72791.692] [batches/s: 0.035] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 03:25:55][utils:57][INFO] [P: 4.00%] [S: 83886080/2097152000] [T: 0:16:54] [ETA: 6:45:43] [loss: 7.151] [tokens/s: 82934.268] [batches/s: 0.040] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 03:25:55][train:194][INFO] Running validation...
+[2025-10-12 03:31:02][logger:171][INFO] [step: 83886080] [val/train_token_count: 83886080] [val/train_batch_count: 40] [val/train_flop_count: 0] [val/train_total_time: 1014.321] [val/train_update_time: 745.889] [val/loss: 7.127] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 306.050] [val/val_tokens_per_second: 133834.286] [val/loss_avg_len_2048: 7.127] [val/perplexity_len_2048: 1245.025] [val/loss_avg_len_1024: 7.128] [val/perplexity_len_1024: 1245.986] [val/loss_avg_len_512: 7.132] [val/perplexity_len_512: 1250.840]
+[2025-10-12 03:33:44][utils:57][INFO] [P: 5.00%] [S: 104857600/2097152000] [T: 0:24:43] [ETA: 7:49:38] [loss: 6.870] [tokens/s: 70627.169] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 03:33:44][logger:171][INFO] [step: 104857600] [train_eval/train_token_count: 104857600] [train_eval/train_batch_count: 50] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 1483.102] [train_eval/train_update_time: 908.402] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 8.243] [train_eval/perplexity_len_2048: 3799.524] [train_eval/loss_avg_len_1024: 8.244] [train_eval/perplexity_len_1024: 3803.436] [train_eval/loss_avg_len_512: 8.245] [train_eval/perplexity_len_512: 3807.527]
+[2025-10-12 03:36:15][utils:57][INFO] [P: 6.00%] [S: 125829120/2097152000] [T: 0:27:13] [ETA: 7:06:37] [loss: 6.583] [tokens/s: 77055.948] [batches/s: 0.037] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 03:36:15][train:194][INFO] Running validation...
+[2025-10-12 03:41:33][logger:171][INFO] [step: 125829120] [val/train_token_count: 125829120] [val/train_batch_count: 60] [val/train_flop_count: 0] [val/train_total_time: 1633.874] [val/train_update_time: 1058.994] [val/loss: 6.576] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 318.172] [val/val_tokens_per_second: 128735.592] [val/loss_avg_len_2048: 6.576] [val/perplexity_len_2048: 717.968] [val/loss_avg_len_1024: 6.578] [val/perplexity_len_1024: 719.126] [val/loss_avg_len_512: 6.584] [val/perplexity_len_512: 723.621]
+[2025-10-12 03:44:04][utils:57][INFO] [P: 7.00%] [S: 146800640/2097152000] [T: 0:35:02] [ETA: 7:45:34] [loss: 6.376] [tokens/s: 69755.793] [batches/s: 0.033] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 03:47:17][utils:57][INFO] [P: 8.00%] [S: 167772160/2097152000] [T: 0:38:16] [ETA: 7:20:04] [loss: 6.187] [tokens/s: 73052.070] [batches/s: 0.035] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 03:47:17][train:194][INFO] Running validation...
+[2025-10-12 03:51:54][logger:171][INFO] [step: 167772160] [val/train_token_count: 167772160] [val/train_batch_count: 80] [val/train_flop_count: 0] [val/train_total_time: 2296.035] [val/train_update_time: 1402.574] [val/loss: 6.158] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 276.364] [val/val_tokens_per_second: 148210.434] [val/loss_avg_len_2048: 6.158] [val/perplexity_len_2048: 472.362] [val/loss_avg_len_1024: 6.160] [val/perplexity_len_1024: 473.592] [val/loss_avg_len_512: 6.169] [val/perplexity_len_512: 477.519]
+[2025-10-12 03:54:40][utils:57][INFO] [P: 9.00%] [S: 188743680/2097152000] [T: 0:45:38] [ETA: 7:41:28] [loss: 6.001] [tokens/s: 68865.792] [batches/s: 0.033] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 03:58:12][utils:57][INFO] [P: 10.00%] [S: 209715200/2097152000] [T: 0:49:10] [ETA: 7:22:36] [loss: 5.889] [tokens/s: 71040.250] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 03:58:12][logger:171][INFO] [step: 209715200] [train_eval/train_token_count: 209715200] [train_eval/train_batch_count: 100] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 2950.673] [train_eval/train_update_time: 1780.236] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 6.304] [train_eval/perplexity_len_2048: 546.867] [train_eval/loss_avg_len_1024: 6.308] [train_eval/perplexity_len_1024: 549.114] [train_eval/loss_avg_len_512: 6.314] [train_eval/perplexity_len_512: 552.415]
+[2025-10-12 03:58:12][train:194][INFO] Running validation...
+[2025-10-12 04:02:15][logger:171][INFO] [step: 209715200] [val/train_token_count: 209715200] [val/train_batch_count: 100] [val/train_flop_count: 0] [val/train_total_time: 2950.673] [val/train_update_time: 1780.236] [val/loss: 5.881] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 242.768] [val/val_tokens_per_second: 168720.497] [val/loss_avg_len_2048: 5.881] [val/perplexity_len_2048: 358.179] [val/loss_avg_len_1024: 5.885] [val/perplexity_len_1024: 359.610] [val/loss_avg_len_512: 5.896] [val/perplexity_len_512: 363.457]
+[2025-10-12 04:02:15][checkpoint:111][INFO] Saving checkpoint to /workspace/forgetting-transformer/alibi_3_4_256/checkpoints/step-000000209715200.pt...
+[2025-10-12 04:02:15][checkpoint:128][INFO] Checkpoint saved to /workspace/forgetting-transformer/alibi_3_4_256/checkpoints/step-000000209715200.pt.
+[2025-10-12 04:02:15][logger:171][INFO] [step: 209715200] [checkpoint/checkpoint_time: 0.565]
+[2025-10-12 04:05:30][utils:57][INFO] [P: 11.00%] [S: 230686720/2097152000] [T: 0:56:28] [ETA: 7:36:57] [loss: 5.765] [tokens/s: 65955.931] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 04:08:57][utils:57][INFO] [P: 12.00%] [S: 251658240/2097152000] [T: 0:59:55] [ETA: 7:19:28] [loss: 5.630] [tokens/s: 71254.046] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 04:08:57][train:194][INFO] Running validation...
+[2025-10-12 04:12:43][logger:171][INFO] [step: 251658240] [val/train_token_count: 251658240] [val/train_batch_count: 120] [val/train_flop_count: 0] [val/train_total_time: 3595.723] [val/train_update_time: 2181.322] [val/loss: 5.645] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 225.882] [val/val_tokens_per_second: 181333.557] [val/loss_avg_len_2048: 5.645] [val/perplexity_len_2048: 282.894] [val/loss_avg_len_1024: 5.651] [val/perplexity_len_1024: 284.492] [val/loss_avg_len_512: 5.664] [val/perplexity_len_512: 288.221]
+[2025-10-12 04:16:15][utils:57][INFO] [P: 13.00%] [S: 272629760/2097152000] [T: 1:07:13] [ETA: 7:29:55] [loss: 5.563] [tokens/s: 65804.287] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 04:19:18][utils:57][INFO] [P: 14.00%] [S: 293601280/2097152000] [T: 1:10:17] [ETA: 7:11:45] [loss: 5.487] [tokens/s: 72200.681] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 04:19:18][train:194][INFO] Running validation...
+[2025-10-12 04:23:34][logger:171][INFO] [step: 293601280] [val/train_token_count: 293601280] [val/train_batch_count: 140] [val/train_flop_count: 0] [val/train_total_time: 4217.221] [val/train_update_time: 2576.439] [val/loss: 5.478] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 255.626] [val/val_tokens_per_second: 160234.000] [val/loss_avg_len_2048: 5.478] [val/perplexity_len_2048: 239.413] [val/loss_avg_len_1024: 5.485] [val/perplexity_len_1024: 240.989] [val/loss_avg_len_512: 5.499] [val/perplexity_len_512: 244.407]
+[2025-10-12 04:27:06][utils:57][INFO] [P: 15.00%] [S: 314572800/2097152000] [T: 1:18:04] [ETA: 7:22:28] [loss: 5.384] [tokens/s: 65150.514] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 04:27:06][logger:171][INFO] [step: 314572800] [train_eval/train_token_count: 314572800] [train_eval/train_batch_count: 150] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 4684.964] [train_eval/train_update_time: 2788.353] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 5.613] [train_eval/perplexity_len_2048: 273.939] [train_eval/loss_avg_len_1024: 5.619] [train_eval/perplexity_len_1024: 275.525] [train_eval/loss_avg_len_512: 5.630] [train_eval/perplexity_len_512: 278.757]
+[2025-10-12 04:29:40][utils:57][INFO] [P: 16.00%] [S: 335544320/2097152000] [T: 1:20:38] [ETA: 7:03:21] [loss: 5.330] [tokens/s: 72308.809] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 04:29:40][train:194][INFO] Running validation...
+[2025-10-12 04:34:32][logger:171][INFO] [step: 335544320] [val/train_token_count: 335544320] [val/train_batch_count: 160] [val/train_flop_count: 0] [val/train_total_time: 4838.394] [val/train_update_time: 2941.477] [val/loss: 5.317] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 292.266] [val/val_tokens_per_second: 140146.108] [val/loss_avg_len_2048: 5.317] [val/perplexity_len_2048: 203.723] [val/loss_avg_len_1024: 5.324] [val/perplexity_len_1024: 205.278] [val/loss_avg_len_512: 5.340] [val/perplexity_len_512: 208.481]
+[2025-10-12 04:37:29][utils:57][INFO] [P: 17.00%] [S: 356515840/2097152000] [T: 1:28:27] [ETA: 7:11:53] [loss: 5.232] [tokens/s: 65084.500] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 04:39:59][utils:57][INFO] [P: 18.00%] [S: 377487360/2097152000] [T: 1:30:57] [ETA: 6:54:21] [loss: 5.203] [tokens/s: 72342.782] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 04:39:59][train:194][INFO] Running validation...
+[2025-10-12 04:45:17][logger:171][INFO] [step: 377487360] [val/train_token_count: 377487360] [val/train_batch_count: 180] [val/train_flop_count: 0] [val/train_total_time: 5457.476] [val/train_update_time: 3267.897] [val/loss: 5.191] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 318.379] [val/val_tokens_per_second: 128651.858] [val/loss_avg_len_2048: 5.191] [val/perplexity_len_2048: 179.674] [val/loss_avg_len_1024: 5.200] [val/perplexity_len_1024: 181.226] [val/loss_avg_len_512: 5.217] [val/perplexity_len_512: 184.331]
+[2025-10-12 04:47:47][utils:57][INFO] [P: 19.00%] [S: 398458880/2097152000] [T: 1:38:46] [ETA: 7:01:04] [loss: 5.190] [tokens/s: 65569.778] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 04:50:46][utils:57][INFO] [P: 20.00%] [S: 419430400/2097152000] [T: 1:41:44] [ETA: 6:46:59] [loss: 5.087] [tokens/s: 71697.200] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 04:50:46][logger:171][INFO] [step: 419430400] [train_eval/train_token_count: 419430400] [train_eval/train_batch_count: 200] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 6104.984] [train_eval/train_update_time: 3596.637] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 5.231] [train_eval/perplexity_len_2048: 187.042] [train_eval/loss_avg_len_1024: 5.240] [train_eval/perplexity_len_1024: 188.688] [train_eval/loss_avg_len_512: 5.256] [train_eval/perplexity_len_512: 191.655]
+[2025-10-12 04:50:46][train:194][INFO] Running validation...
+[2025-10-12 04:55:37][logger:171][INFO] [step: 419430400] [val/train_token_count: 419430400] [val/train_batch_count: 200] [val/train_flop_count: 0] [val/train_total_time: 6104.984] [val/train_update_time: 3596.637] [val/loss: 5.086] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 290.458] [val/val_tokens_per_second: 141018.895] [val/loss_avg_len_2048: 5.086] [val/perplexity_len_2048: 161.794] [val/loss_avg_len_1024: 5.096] [val/perplexity_len_1024: 163.325] [val/loss_avg_len_512: 5.114] [val/perplexity_len_512: 166.367]
+[2025-10-12 04:55:37][checkpoint:111][INFO] Saving checkpoint to /workspace/forgetting-transformer/alibi_3_4_256/checkpoints/step-000000419430400.pt...
+[2025-10-12 04:55:37][checkpoint:128][INFO] Checkpoint saved to /workspace/forgetting-transformer/alibi_3_4_256/checkpoints/step-000000419430400.pt.
+[2025-10-12 04:55:37][logger:171][INFO] [step: 419430400] [checkpoint/checkpoint_time: 0.567]
+[2025-10-12 04:58:12][utils:57][INFO] [P: 21.00%] [S: 440401920/2097152000] [T: 1:49:10] [ETA: 6:50:42] [loss: 5.045] [tokens/s: 66113.362] [batches/s: 0.032] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 05:01:44][utils:57][INFO] [P: 22.00%] [S: 461373440/2097152000] [T: 1:52:42] [ETA: 6:39:36] [loss: 5.003] [tokens/s: 71107.822] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 05:01:44][train:194][INFO] Running validation...
+[2025-10-12 05:05:58][logger:171][INFO] [step: 461373440] [val/train_token_count: 461373440] [val/train_batch_count: 220] [val/train_flop_count: 0] [val/train_total_time: 6762.652] [val/train_update_time: 3962.670] [val/loss: 4.994] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 254.082] [val/val_tokens_per_second: 161207.486] [val/loss_avg_len_2048: 4.994] [val/perplexity_len_2048: 147.597] [val/loss_avg_len_1024: 5.005] [val/perplexity_len_1024: 149.120] [val/loss_avg_len_512: 5.025] [val/perplexity_len_512: 152.132]
+[2025-10-12 05:09:03][utils:57][INFO] [P: 23.00%] [S: 482344960/2097152000] [T: 2:00:01] [ETA: 6:41:49] [loss: 4.950] [tokens/s: 65984.742] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 05:12:35][utils:57][INFO] [P: 24.00%] [S: 503316480/2097152000] [T: 2:03:33] [ETA: 6:31:17] [loss: 4.899] [tokens/s: 71106.395] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 05:12:35][train:194][INFO] Running validation...
+[2025-10-12 05:16:20][logger:171][INFO] [step: 503316480] [val/train_token_count: 503316480] [val/train_batch_count: 240] [val/train_flop_count: 0] [val/train_total_time: 7413.931] [val/train_update_time: 4359.250] [val/loss: 4.918] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 224.986] [val/val_tokens_per_second: 182055.460] [val/loss_avg_len_2048: 4.918] [val/perplexity_len_2048: 136.664] [val/loss_avg_len_1024: 4.929] [val/perplexity_len_1024: 138.195] [val/loss_avg_len_512: 4.950] [val/perplexity_len_512: 141.180]
+[2025-10-12 05:19:49][utils:57][INFO] [P: 25.00%] [S: 524288000/2097152000] [T: 2:10:47] [ETA: 6:32:23] [loss: 4.893] [tokens/s: 66018.291] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 05:19:49][logger:171][INFO] [step: 524288000] [train_eval/train_token_count: 524288000] [train_eval/train_batch_count: 250] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 7847.669] [train_eval/train_update_time: 4567.680] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.981] [train_eval/perplexity_len_2048: 145.578] [train_eval/loss_avg_len_1024: 4.989] [train_eval/perplexity_len_1024: 146.857] [train_eval/loss_avg_len_512: 5.008] [train_eval/perplexity_len_512: 149.576]
+[2025-10-12 05:23:01][utils:57][INFO] [P: 26.00%] [S: 545259520/2097152000] [T: 2:14:00] [ETA: 6:21:23] [loss: 4.843] [tokens/s: 71880.913] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 05:23:01][train:194][INFO] Running validation...
+[2025-10-12 05:27:05][logger:171][INFO] [step: 545259520] [val/train_token_count: 545259520] [val/train_batch_count: 260] [val/train_flop_count: 0] [val/train_total_time: 8040.308] [val/train_update_time: 4760.006] [val/loss: 4.846] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 243.840] [val/val_tokens_per_second: 167979.168] [val/loss_avg_len_2048: 4.846] [val/perplexity_len_2048: 127.282] [val/loss_avg_len_1024: 4.858] [val/perplexity_len_1024: 128.807] [val/loss_avg_len_512: 4.881] [val/perplexity_len_512: 131.766]
+[2025-10-12 05:30:38][utils:57][INFO] [P: 27.00%] [S: 566231040/2097152000] [T: 2:21:36] [ETA: 6:22:51] [loss: 4.826] [tokens/s: 65416.845] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 05:33:23][utils:57][INFO] [P: 28.00%] [S: 587202560/2097152000] [T: 2:24:21] [ETA: 6:11:12] [loss: 4.774] [tokens/s: 72327.444] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 05:33:23][train:194][INFO] Running validation...
+[2025-10-12 05:38:00][logger:171][INFO] [step: 587202560] [val/train_token_count: 587202560] [val/train_batch_count: 280] [val/train_flop_count: 0] [val/train_total_time: 8661.466] [val/train_update_time: 5136.806] [val/loss: 4.786] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 277.761] [val/val_tokens_per_second: 147464.894] [val/loss_avg_len_2048: 4.786] [val/perplexity_len_2048: 119.808] [val/loss_avg_len_1024: 4.799] [val/perplexity_len_1024: 121.397] [val/loss_avg_len_512: 4.824] [val/perplexity_len_512: 124.424]
+[2025-10-12 05:41:13][utils:57][INFO] [P: 29.00%] [S: 608174080/2097152000] [T: 2:32:11] [ETA: 6:12:36] [loss: 4.738] [tokens/s: 65081.466] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 05:43:43][utils:57][INFO] [P: 30.00%] [S: 629145600/2097152000] [T: 2:34:41] [ETA: 6:00:57] [loss: 4.711] [tokens/s: 72329.630] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 05:43:43][logger:171][INFO] [step: 629145600] [train_eval/train_token_count: 629145600] [train_eval/train_batch_count: 300] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 9281.589] [train_eval/train_update_time: 5478.767] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.806] [train_eval/perplexity_len_2048: 122.237] [train_eval/loss_avg_len_1024: 4.816] [train_eval/perplexity_len_1024: 123.415] [train_eval/loss_avg_len_512: 4.837] [train_eval/perplexity_len_512: 126.092]
+[2025-10-12 05:43:43][train:194][INFO] Running validation...
+[2025-10-12 05:49:01][logger:171][INFO] [step: 629145600] [val/train_token_count: 629145600] [val/train_batch_count: 300] [val/train_flop_count: 0] [val/train_total_time: 9281.589] [val/train_update_time: 5478.767] [val/loss: 4.726] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 318.064] [val/val_tokens_per_second: 128778.950] [val/loss_avg_len_2048: 4.726] [val/perplexity_len_2048: 112.849] [val/loss_avg_len_1024: 4.740] [val/perplexity_len_1024: 114.465] [val/loss_avg_len_512: 4.767] [val/perplexity_len_512: 117.556]
+[2025-10-12 05:49:01][checkpoint:111][INFO] Saving checkpoint to /workspace/forgetting-transformer/alibi_3_4_256/checkpoints/step-000000629145600.pt...
+[2025-10-12 05:49:01][checkpoint:128][INFO] Checkpoint saved to /workspace/forgetting-transformer/alibi_3_4_256/checkpoints/step-000000629145600.pt.
+[2025-10-12 05:49:01][logger:171][INFO] [step: 629145600] [checkpoint/checkpoint_time: 0.564]
+[2025-10-12 05:51:32][utils:57][INFO] [P: 31.00%] [S: 650117120/2097152000] [T: 2:42:30] [ETA: 6:01:42] [loss: 4.730] [tokens/s: 65312.771] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 05:54:16][utils:57][INFO] [P: 32.00%] [S: 671088640/2097152000] [T: 2:45:14] [ETA: 5:51:08] [loss: 4.656] [tokens/s: 72021.946] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 05:54:16][train:194][INFO] Running validation...
+[2025-10-12 05:59:21][logger:171][INFO] [step: 671088640] [val/train_token_count: 671088640] [val/train_batch_count: 320] [val/train_flop_count: 0] [val/train_total_time: 9914.600] [val/train_update_time: 5792.780] [val/loss: 4.671] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 304.963] [val/val_tokens_per_second: 134311.182] [val/loss_avg_len_2048: 4.671] [val/perplexity_len_2048: 106.798] [val/loss_avg_len_1024: 4.686] [val/perplexity_len_1024: 108.428] [val/loss_avg_len_512: 4.714] [val/perplexity_len_512: 111.516]
+[2025-10-12 06:01:51][utils:57][INFO] [P: 33.00%] [S: 692060160/2097152000] [T: 2:52:49] [ETA: 5:50:53] [loss: 4.678] [tokens/s: 65982.383] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 06:05:15][utils:57][INFO] [P: 34.00%] [S: 713031680/2097152000] [T: 2:56:14] [ETA: 5:42:06] [loss: 4.628] [tokens/s: 71166.432] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 06:05:15][train:194][INFO] Running validation...
+[2025-10-12 06:09:41][logger:171][INFO] [step: 713031680] [val/train_token_count: 713031680] [val/train_batch_count: 340] [val/train_flop_count: 0] [val/train_total_time: 10574.152] [val/train_update_time: 6146.868] [val/loss: 4.621] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 265.643] [val/val_tokens_per_second: 154191.823] [val/loss_avg_len_2048: 4.621] [val/perplexity_len_2048: 101.562] [val/loss_avg_len_1024: 4.637] [val/perplexity_len_1024: 103.235] [val/loss_avg_len_512: 4.667] [val/perplexity_len_512: 106.381]
+[2025-10-12 06:12:36][utils:57][INFO] [P: 35.00%] [S: 734003200/2097152000] [T: 3:03:34] [ETA: 5:40:55] [loss: 4.613] [tokens/s: 66004.529] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 06:12:36][logger:171][INFO] [step: 734003200] [train_eval/train_token_count: 734003200] [train_eval/train_batch_count: 350] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 11014.551] [train_eval/train_update_time: 6321.313] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.663] [train_eval/perplexity_len_2048: 106.004] [train_eval/loss_avg_len_1024: 4.678] [train_eval/perplexity_len_1024: 107.509] [train_eval/loss_avg_len_512: 4.704] [train_eval/perplexity_len_512: 110.432]
+[2025-10-12 06:16:08][utils:57][INFO] [P: 36.00%] [S: 754974720/2097152000] [T: 3:07:06] [ETA: 5:32:38] [loss: 4.533] [tokens/s: 71066.048] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 06:16:08][train:194][INFO] Running validation...
+[2025-10-12 06:20:02][logger:171][INFO] [step: 754974720] [val/train_token_count: 754974720] [val/train_batch_count: 360] [val/train_flop_count: 0] [val/train_total_time: 11226.902] [val/train_update_time: 6533.353] [val/loss: 4.575] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 234.033] [val/val_tokens_per_second: 175017.940] [val/loss_avg_len_2048: 4.575] [val/perplexity_len_2048: 97.058] [val/loss_avg_len_1024: 4.593] [val/perplexity_len_1024: 98.782] [val/loss_avg_len_512: 4.625] [val/perplexity_len_512: 101.985]
+[2025-10-12 06:23:23][utils:57][INFO] [P: 37.00%] [S: 775946240/2097152000] [T: 3:14:21] [ETA: 5:30:56] [loss: 4.552] [tokens/s: 66031.089] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 06:26:44][utils:57][INFO] [P: 38.00%] [S: 796917760/2097152000] [T: 3:17:42] [ETA: 5:22:34] [loss: 4.527] [tokens/s: 71543.755] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 06:26:44][train:194][INFO] Running validation...
+[2025-10-12 06:30:38][logger:171][INFO] [step: 796917760] [val/train_token_count: 796917760] [val/train_batch_count: 380] [val/train_flop_count: 0] [val/train_total_time: 11862.473] [val/train_update_time: 6934.268] [val/loss: 4.536] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 234.151] [val/val_tokens_per_second: 174929.848] [val/loss_avg_len_2048: 4.536] [val/perplexity_len_2048: 93.335] [val/loss_avg_len_1024: 4.555] [val/perplexity_len_1024: 95.118] [val/loss_avg_len_512: 4.589] [val/perplexity_len_512: 98.420]
+[2025-10-12 06:34:10][utils:57][INFO] [P: 39.00%] [S: 817889280/2097152000] [T: 3:25:08] [ETA: 5:20:52] [loss: 4.524] [tokens/s: 65652.130] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 06:37:05][utils:57][INFO] [P: 40.00%] [S: 838860800/2097152000] [T: 3:28:03] [ETA: 5:12:05] [loss: 4.428] [tokens/s: 72376.710] [batches/s: 0.035] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 06:37:05][logger:171][INFO] [step: 838860800] [train_eval/train_token_count: 838860800] [train_eval/train_batch_count: 400] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 12483.889] [train_eval/train_update_time: 7321.040] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.545] [train_eval/perplexity_len_2048: 94.196] [train_eval/loss_avg_len_1024: 4.561] [train_eval/perplexity_len_1024: 95.677] [train_eval/loss_avg_len_512: 4.593] [train_eval/perplexity_len_512: 98.785]
+[2025-10-12 06:37:05][train:194][INFO] Running validation...
+[2025-10-12 06:41:31][logger:171][INFO] [step: 838860800] [val/train_token_count: 838860800] [val/train_batch_count: 400] [val/train_flop_count: 0] [val/train_total_time: 12483.889] [val/train_update_time: 7321.040] [val/loss: 4.494] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 265.758] [val/val_tokens_per_second: 154125.171] [val/loss_avg_len_2048: 4.494] [val/perplexity_len_2048: 89.461] [val/loss_avg_len_1024: 4.514] [val/perplexity_len_1024: 91.311] [val/loss_avg_len_512: 4.551] [val/perplexity_len_512: 94.706]
+[2025-10-12 06:41:31][checkpoint:111][INFO] Saving checkpoint to /workspace/forgetting-transformer/alibi_3_4_256/checkpoints/step-000000838860800.pt...
+[2025-10-12 06:41:31][checkpoint:128][INFO] Checkpoint saved to /workspace/forgetting-transformer/alibi_3_4_256/checkpoints/step-000000838860800.pt.
+[2025-10-12 06:41:31][logger:171][INFO] [step: 838860800] [checkpoint/checkpoint_time: 0.486]
+[2025-10-12 06:44:56][utils:57][INFO] [P: 41.00%] [S: 859832320/2097152000] [T: 3:35:54] [ETA: 5:10:41] [loss: 4.447] [tokens/s: 65108.785] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 06:47:26][utils:57][INFO] [P: 42.00%] [S: 880803840/2097152000] [T: 3:38:24] [ETA: 5:01:36] [loss: 4.442] [tokens/s: 72347.206] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 06:47:26][train:194][INFO] Running validation...
+[2025-10-12 06:52:31][logger:171][INFO] [step: 880803840] [val/train_token_count: 880803840] [val/train_batch_count: 420] [val/train_flop_count: 0] [val/train_total_time: 13104.464] [val/train_update_time: 7674.995] [val/loss: 4.453] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 305.227] [val/val_tokens_per_second: 134195.041] [val/loss_avg_len_2048: 4.453] [val/perplexity_len_2048: 85.913] [val/loss_avg_len_1024: 4.476] [val/perplexity_len_1024: 87.843] [val/loss_avg_len_512: 4.515] [val/perplexity_len_512: 91.353]
+[2025-10-12 06:55:14][utils:57][INFO] [P: 43.00%] [S: 901775360/2097152000] [T: 3:46:12] [ETA: 4:59:51] [loss: 4.436] [tokens/s: 65119.378] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 06:57:43][utils:57][INFO] [P: 44.00%] [S: 922746880/2097152000] [T: 3:48:41] [ETA: 4:51:04] [loss: 4.456] [tokens/s: 72413.812] [batches/s: 0.035] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 06:57:43][train:194][INFO] Running validation...
+[2025-10-12 07:03:01][logger:171][INFO] [step: 922746880] [val/train_token_count: 922746880] [val/train_batch_count: 440] [val/train_flop_count: 0] [val/train_total_time: 13721.995] [val/train_update_time: 7986.941] [val/loss: 4.414] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 318.030] [val/val_tokens_per_second: 128793.007] [val/loss_avg_len_2048: 4.414] [val/perplexity_len_2048: 82.592] [val/loss_avg_len_1024: 4.438] [val/perplexity_len_1024: 84.598] [val/loss_avg_len_512: 4.480] [val/perplexity_len_512: 88.242]
+[2025-10-12 07:05:31][utils:57][INFO] [P: 45.00%] [S: 943718400/2097152000] [T: 3:56:29] [ETA: 4:49:03] [loss: 4.393] [tokens/s: 65828.089] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 07:05:31][logger:171][INFO] [step: 943718400] [train_eval/train_token_count: 943718400] [train_eval/train_batch_count: 450] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 14189.862] [train_eval/train_update_time: 8136.603] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.442] [train_eval/perplexity_len_2048: 84.986] [train_eval/loss_avg_len_1024: 4.464] [train_eval/perplexity_len_1024: 86.869] [train_eval/loss_avg_len_512: 4.504] [train_eval/perplexity_len_512: 90.406]
+[2025-10-12 07:08:43][utils:57][INFO] [P: 46.00%] [S: 964689920/2097152000] [T: 3:59:41] [ETA: 4:41:22] [loss: 4.356] [tokens/s: 71452.877] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 07:08:43][train:194][INFO] Running validation...
+[2025-10-12 07:13:21][logger:171][INFO] [step: 964689920] [val/train_token_count: 964689920] [val/train_batch_count: 460] [val/train_flop_count: 0] [val/train_total_time: 14381.773] [val/train_update_time: 8328.335] [val/loss: 4.376] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 277.731] [val/val_tokens_per_second: 147480.975] [val/loss_avg_len_2048: 4.376] [val/perplexity_len_2048: 79.490] [val/loss_avg_len_1024: 4.401] [val/perplexity_len_1024: 81.572] [val/loss_avg_len_512: 4.447] [val/perplexity_len_512: 85.332]
+[2025-10-12 07:16:05][utils:57][INFO] [P: 47.00%] [S: 985661440/2097152000] [T: 4:07:03] [ETA: 4:38:35] [loss: 4.363] [tokens/s: 66114.739] [batches/s: 0.032] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 07:19:37][utils:57][INFO] [P: 48.00%] [S: 1006632960/2097152000] [T: 4:10:35] [ETA: 4:31:28] [loss: 4.340] [tokens/s: 71151.326] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 07:19:37][train:194][INFO] Running validation...
+[2025-10-12 07:23:42][logger:171][INFO] [step: 1006632960] [val/train_token_count: 1006632960] [val/train_batch_count: 480] [val/train_flop_count: 0] [val/train_total_time: 15035.863] [val/train_update_time: 8704.088] [val/loss: 4.340] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 244.463] [val/val_tokens_per_second: 167550.746] [val/loss_avg_len_2048: 4.340] [val/perplexity_len_2048: 76.689] [val/loss_avg_len_1024: 4.367] [val/perplexity_len_1024: 78.842] [val/loss_avg_len_512: 4.416] [val/perplexity_len_512: 82.724]
+[2025-10-12 07:26:54][utils:57][INFO] [P: 49.00%] [S: 1027604480/2097152000] [T: 4:17:53] [ETA: 4:28:24] [loss: 4.335] [tokens/s: 66059.000] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 07:30:23][utils:57][INFO] [P: 50.00%] [S: 1048576000/2097152000] [T: 4:21:21] [ETA: 4:21:21] [loss: 4.304] [tokens/s: 71334.309] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 07:30:23][logger:171][INFO] [step: 1048576000] [train_eval/train_token_count: 1048576000] [train_eval/train_batch_count: 500] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 15681.829] [train_eval/train_update_time: 9104.970] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.350] [train_eval/perplexity_len_2048: 77.460] [train_eval/loss_avg_len_1024: 4.374] [train_eval/perplexity_len_1024: 79.345] [train_eval/loss_avg_len_512: 4.421] [train_eval/perplexity_len_512: 83.144]
+[2025-10-12 07:30:23][train:194][INFO] Running validation...
+[2025-10-12 07:34:08][logger:171][INFO] [step: 1048576000] [val/train_token_count: 1048576000] [val/train_batch_count: 500] [val/train_flop_count: 0] [val/train_total_time: 15681.829] [val/train_update_time: 9104.970] [val/loss: 4.306] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 225.085] [val/val_tokens_per_second: 181975.673] [val/loss_avg_len_2048: 4.306] [val/perplexity_len_2048: 74.174] [val/loss_avg_len_1024: 4.336] [val/perplexity_len_1024: 76.368] [val/loss_avg_len_512: 4.386] [val/perplexity_len_512: 80.324]
+[2025-10-12 07:34:08][checkpoint:111][INFO] Saving checkpoint to /workspace/forgetting-transformer/alibi_3_4_256/checkpoints/step-000001048576000.pt...
+[2025-10-12 07:34:09][checkpoint:128][INFO] Checkpoint saved to /workspace/forgetting-transformer/alibi_3_4_256/checkpoints/step-000001048576000.pt.
+[2025-10-12 07:34:09][logger:171][INFO] [step: 1048576000] [checkpoint/checkpoint_time: 0.474]
+[2025-10-12 07:37:41][utils:57][INFO] [P: 51.00%] [S: 1069547520/2097152000] [T: 4:28:39] [ETA: 4:18:07] [loss: 4.291] [tokens/s: 65907.600] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 07:40:45][utils:57][INFO] [P: 52.00%] [S: 1090519040/2097152000] [T: 4:31:43] [ETA: 4:10:49] [loss: 4.283] [tokens/s: 72269.375] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 07:40:45][train:194][INFO] Running validation...
+[2025-10-12 07:44:59][logger:171][INFO] [step: 1090519040] [val/train_token_count: 1090519040] [val/train_batch_count: 520] [val/train_flop_count: 0] [val/train_total_time: 16303.791] [val/train_update_time: 9500.859] [val/loss: 4.275] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 254.239] [val/val_tokens_per_second: 161108.029] [val/loss_avg_len_2048: 4.275] [val/perplexity_len_2048: 71.868] [val/loss_avg_len_1024: 4.306] [val/perplexity_len_1024: 74.164] [val/loss_avg_len_512: 4.360] [val/perplexity_len_512: 78.271]
+[2025-10-12 07:48:31][utils:57][INFO] [P: 53.00%] [S: 1111490560/2097152000] [T: 4:39:30] [ETA: 4:07:51] [loss: 4.243] [tokens/s: 65239.302] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 07:51:05][utils:57][INFO] [P: 54.00%] [S: 1132462080/2097152000] [T: 4:42:04] [ETA: 4:00:16] [loss: 4.252] [tokens/s: 72358.782] [batches/s: 0.035] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 07:51:05][train:194][INFO] Running validation...
+[2025-10-12 07:55:56][logger:171][INFO] [step: 1132462080] [val/train_token_count: 1132462080] [val/train_batch_count: 540] [val/train_flop_count: 0] [val/train_total_time: 16924.294] [val/train_update_time: 9866.632] [val/loss: 4.246] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 290.549] [val/val_tokens_per_second: 140974.312] [val/loss_avg_len_2048: 4.246] [val/perplexity_len_2048: 69.851] [val/loss_avg_len_1024: 4.280] [val/perplexity_len_1024: 72.224] [val/loss_avg_len_512: 4.336] [val/perplexity_len_512: 76.417]
+[2025-10-12 07:58:55][utils:57][INFO] [P: 55.00%] [S: 1153433600/2097152000] [T: 4:49:53] [ETA: 3:57:10] [loss: 4.190] [tokens/s: 65114.053] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 07:58:55][logger:171][INFO] [step: 1153433600] [train_eval/train_token_count: 1153433600] [train_eval/train_batch_count: 550] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 17393.371] [train_eval/train_update_time: 10044.969] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.264] [train_eval/perplexity_len_2048: 71.063] [train_eval/loss_avg_len_1024: 4.291] [train_eval/perplexity_len_1024: 73.003] [train_eval/loss_avg_len_512: 4.342] [train_eval/perplexity_len_512: 76.851]
+[2025-10-12 08:01:24][utils:57][INFO] [P: 56.00%] [S: 1174405120/2097152000] [T: 4:52:23] [ETA: 3:49:43] [loss: 4.234] [tokens/s: 72379.532] [batches/s: 0.035] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 08:01:24][train:194][INFO] Running validation...
+[2025-10-12 08:06:42][logger:171][INFO] [step: 1174405120] [val/train_token_count: 1174405120] [val/train_batch_count: 560] [val/train_flop_count: 0] [val/train_total_time: 17543.089] [val/train_update_time: 10194.514] [val/loss: 4.217] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 317.977] [val/val_tokens_per_second: 128814.410] [val/loss_avg_len_2048: 4.217] [val/perplexity_len_2048: 67.804] [val/loss_avg_len_1024: 4.251] [val/perplexity_len_1024: 70.198] [val/loss_avg_len_512: 4.310] [val/perplexity_len_512: 74.435]
+[2025-10-12 08:09:12][utils:57][INFO] [P: 57.00%] [S: 1195376640/2097152000] [T: 5:00:10] [ETA: 3:46:27] [loss: 4.166] [tokens/s: 65580.496] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 08:12:08][utils:57][INFO] [P: 58.00%] [S: 1216348160/2097152000] [T: 5:03:06] [ETA: 3:39:29] [loss: 4.223] [tokens/s: 71802.232] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 08:12:08][train:194][INFO] Running validation...
+[2025-10-12 08:17:01][logger:171][INFO] [step: 1216348160] [val/train_token_count: 1216348160] [val/train_batch_count: 580] [val/train_flop_count: 0] [val/train_total_time: 18186.985] [val/train_update_time: 10520.075] [val/loss: 4.191] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 292.995] [val/val_tokens_per_second: 139797.638] [val/loss_avg_len_2048: 4.191] [val/perplexity_len_2048: 66.088] [val/loss_avg_len_1024: 4.228] [val/perplexity_len_1024: 68.556] [val/loss_avg_len_512: 4.289] [val/perplexity_len_512: 72.913]
+[2025-10-12 08:19:33][utils:57][INFO] [P: 59.00%] [S: 1237319680/2097152000] [T: 5:10:32] [ETA: 3:35:47] [loss: 4.212] [tokens/s: 66171.777] [batches/s: 0.032] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 08:23:06][utils:57][INFO] [P: 60.00%] [S: 1258291200/2097152000] [T: 5:14:04] [ETA: 3:29:22] [loss: 4.208] [tokens/s: 71206.800] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 08:23:06][logger:171][INFO] [step: 1258291200] [train_eval/train_token_count: 1258291200] [train_eval/train_batch_count: 600] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 18844.379] [train_eval/train_update_time: 10883.880] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.193] [train_eval/perplexity_len_2048: 66.222] [train_eval/loss_avg_len_1024: 4.223] [train_eval/perplexity_len_1024: 68.232] [train_eval/loss_avg_len_512: 4.282] [train_eval/perplexity_len_512: 72.390]
+[2025-10-12 08:23:06][train:194][INFO] Running validation...
+[2025-10-12 08:27:22][logger:171][INFO] [step: 1258291200] [val/train_token_count: 1258291200] [val/train_batch_count: 600] [val/train_flop_count: 0] [val/train_total_time: 18844.379] [val/train_update_time: 10883.880] [val/loss: 4.169] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 256.153] [val/val_tokens_per_second: 159904.451] [val/loss_avg_len_2048: 4.169] [val/perplexity_len_2048: 64.671] [val/loss_avg_len_1024: 4.207] [val/perplexity_len_1024: 67.139] [val/loss_avg_len_512: 4.270] [val/perplexity_len_512: 71.504]
+[2025-10-12 08:27:22][checkpoint:111][INFO] Saving checkpoint to /workspace/forgetting-transformer/alibi_3_4_256/checkpoints/step-000001258291200.pt...
+[2025-10-12 08:27:22][checkpoint:128][INFO] Checkpoint saved to /workspace/forgetting-transformer/alibi_3_4_256/checkpoints/step-000001258291200.pt.
+[2025-10-12 08:27:22][logger:171][INFO] [step: 1258291200] [checkpoint/checkpoint_time: 0.555]
+[2025-10-12 08:30:26][utils:57][INFO] [P: 61.00%] [S: 1279262720/2097152000] [T: 5:21:24] [ETA: 3:25:29] [loss: 4.168] [tokens/s: 66048.434] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 08:33:58][utils:57][INFO] [P: 62.00%] [S: 1300234240/2097152000] [T: 5:24:56] [ETA: 3:19:09] [loss: 4.140] [tokens/s: 71165.810] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 08:33:58][train:194][INFO] Running validation...
+[2025-10-12 08:37:44][logger:171][INFO] [step: 1300234240] [val/train_token_count: 1300234240] [val/train_batch_count: 620] [val/train_flop_count: 0] [val/train_total_time: 19496.671] [val/train_update_time: 11278.836] [val/loss: 4.146] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 226.110] [val/val_tokens_per_second: 181150.523] [val/loss_avg_len_2048: 4.146] [val/perplexity_len_2048: 63.197] [val/loss_avg_len_1024: 4.186] [val/perplexity_len_1024: 65.746] [val/loss_avg_len_512: 4.251] [val/perplexity_len_512: 70.201]
+[2025-10-12 08:41:12][utils:57][INFO] [P: 63.00%] [S: 1321205760/2097152000] [T: 5:32:10] [ETA: 3:15:05] [loss: 4.151] [tokens/s: 66098.394] [batches/s: 0.032] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 08:44:25][utils:57][INFO] [P: 64.00%] [S: 1342177280/2097152000] [T: 5:35:24] [ETA: 3:08:39] [loss: 4.140] [tokens/s: 71887.073] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 08:44:25][train:194][INFO] Running validation...
+[2025-10-12 08:48:28][logger:171][INFO] [step: 1342177280] [val/train_token_count: 1342177280] [val/train_batch_count: 640] [val/train_flop_count: 0] [val/train_total_time: 20124.233] [val/train_update_time: 11679.664] [val/loss: 4.127] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 242.415] [val/val_tokens_per_second: 168966.614] [val/loss_avg_len_2048: 4.127] [val/perplexity_len_2048: 61.972] [val/loss_avg_len_1024: 4.167] [val/perplexity_len_1024: 64.531] [val/loss_avg_len_512: 4.234] [val/perplexity_len_512: 69.015]
+[2025-10-12 08:52:00][utils:57][INFO] [P: 65.00%] [S: 1363148800/2097152000] [T: 5:42:58] [ETA: 3:04:40] [loss: 4.111] [tokens/s: 65483.584] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 08:52:00][logger:171][INFO] [step: 1363148800] [train_eval/train_token_count: 1363148800] [train_eval/train_batch_count: 650] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 20578.881] [train_eval/train_update_time: 11891.677] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.141] [train_eval/perplexity_len_2048: 62.883] [train_eval/loss_avg_len_1024: 4.179] [train_eval/perplexity_len_1024: 65.299] [train_eval/loss_avg_len_512: 4.243] [train_eval/perplexity_len_512: 69.628]
+[2025-10-12 08:54:46][utils:57][INFO] [P: 66.00%] [S: 1384120320/2097152000] [T: 5:45:44] [ETA: 2:58:06] [loss: 4.139] [tokens/s: 72369.317] [batches/s: 0.035] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 08:54:46][train:194][INFO] Running validation...
+[2025-10-12 08:59:22][logger:171][INFO] [step: 1384120320] [val/train_token_count: 1384120320] [val/train_batch_count: 660] [val/train_flop_count: 0] [val/train_total_time: 20744.930] [val/train_update_time: 12057.428] [val/loss: 4.109] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 275.830] [val/val_tokens_per_second: 148497.049] [val/loss_avg_len_2048: 4.109] [val/perplexity_len_2048: 60.878] [val/loss_avg_len_1024: 4.151] [val/perplexity_len_1024: 63.485] [val/loss_avg_len_512: 4.220] [val/perplexity_len_512: 68.012]
+[2025-10-12 09:02:36][utils:57][INFO] [P: 67.00%] [S: 1405091840/2097152000] [T: 5:53:34] [ETA: 2:54:09] [loss: 4.076] [tokens/s: 65104.314] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 09:05:06][utils:57][INFO] [P: 68.00%] [S: 1426063360/2097152000] [T: 5:56:04] [ETA: 2:47:33] [loss: 4.093] [tokens/s: 72358.293] [batches/s: 0.035] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 09:05:06][train:194][INFO] Running validation...
+[2025-10-12 09:10:23][logger:171][INFO] [step: 1426063360] [val/train_token_count: 1426063360] [val/train_batch_count: 680] [val/train_flop_count: 0] [val/train_total_time: 21364.400] [val/train_update_time: 12400.716] [val/loss: 4.094] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 317.506] [val/val_tokens_per_second: 129005.366] [val/loss_avg_len_2048: 4.094] [val/perplexity_len_2048: 59.961] [val/loss_avg_len_1024: 4.137] [val/perplexity_len_1024: 62.590] [val/loss_avg_len_512: 4.207] [val/perplexity_len_512: 67.153]
+[2025-10-12 09:12:53][utils:57][INFO] [P: 69.00%] [S: 1447034880/2097152000] [T: 6:03:51] [ETA: 2:43:28] [loss: 4.098] [tokens/s: 65320.048] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 09:15:34][utils:57][INFO] [P: 70.00%] [S: 1468006400/2097152000] [T: 6:06:32] [ETA: 2:37:05] [loss: 4.090] [tokens/s: 72184.772] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 09:15:34][logger:171][INFO] [step: 1468006400] [train_eval/train_token_count: 1468006400] [train_eval/train_batch_count: 700] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 21992.386] [train_eval/train_update_time: 12710.862] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.095] [train_eval/perplexity_len_2048: 60.056] [train_eval/loss_avg_len_1024: 4.135] [train_eval/perplexity_len_1024: 62.515] [train_eval/loss_avg_len_512: 4.205] [train_eval/perplexity_len_512: 66.994]
+[2025-10-12 09:15:34][train:194][INFO] Running validation...
+[2025-10-12 09:20:41][logger:171][INFO] [step: 1468006400] [val/train_token_count: 1468006400] [val/train_batch_count: 700] [val/train_flop_count: 0] [val/train_total_time: 21992.386] [val/train_update_time: 12710.862] [val/loss: 4.080] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 307.703] [val/val_tokens_per_second: 133115.201] [val/loss_avg_len_2048: 4.080] [val/perplexity_len_2048: 59.132] [val/loss_avg_len_1024: 4.123] [val/perplexity_len_1024: 61.747] [val/loss_avg_len_512: 4.194] [val/perplexity_len_512: 66.315]
+[2025-10-12 09:20:41][checkpoint:111][INFO] Saving checkpoint to /workspace/forgetting-transformer/alibi_3_4_256/checkpoints/step-000001468006400.pt...
+[2025-10-12 09:20:42][checkpoint:128][INFO] Checkpoint saved to /workspace/forgetting-transformer/alibi_3_4_256/checkpoints/step-000001468006400.pt.
+[2025-10-12 09:20:42][logger:171][INFO] [step: 1468006400] [checkpoint/checkpoint_time: 0.571]
+[2025-10-12 09:23:11][utils:57][INFO] [P: 71.00%] [S: 1488977920/2097152000] [T: 6:14:10] [ETA: 2:32:49] [loss: 4.094] [tokens/s: 66028.790] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 09:26:34][utils:57][INFO] [P: 72.00%] [S: 1509949440/2097152000] [T: 6:17:33] [ETA: 2:26:49] [loss: 4.064] [tokens/s: 71263.566] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 09:26:34][train:194][INFO] Running validation...
+[2025-10-12 09:31:02][logger:171][INFO] [step: 1509949440] [val/train_token_count: 1509949440] [val/train_batch_count: 720] [val/train_flop_count: 0] [val/train_total_time: 22653.045] [val/train_update_time: 13062.763] [val/loss: 4.068] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 267.438] [val/val_tokens_per_second: 153157.192] [val/loss_avg_len_2048: 4.068] [val/perplexity_len_2048: 58.434] [val/loss_avg_len_1024: 4.111] [val/perplexity_len_1024: 61.037] [val/loss_avg_len_512: 4.184] [val/perplexity_len_512: 65.607]
+[2025-10-12 09:33:55][utils:57][INFO] [P: 73.00%] [S: 1530920960/2097152000] [T: 6:24:53] [ETA: 2:22:21] [loss: 4.084] [tokens/s: 66087.505] [batches/s: 0.032] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 09:37:27][utils:57][INFO] [P: 74.00%] [S: 1551892480/2097152000] [T: 6:28:25] [ETA: 2:16:28] [loss: 4.059] [tokens/s: 71151.478] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 09:37:27][train:194][INFO] Running validation...
+[2025-10-12 09:41:23][logger:171][INFO] [step: 1551892480] [val/train_token_count: 1551892480] [val/train_batch_count: 740] [val/train_flop_count: 0] [val/train_total_time: 23305.888] [val/train_update_time: 13447.545] [val/loss: 4.056] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 235.905] [val/val_tokens_per_second: 173628.964] [val/loss_avg_len_2048: 4.056] [val/perplexity_len_2048: 57.765] [val/loss_avg_len_1024: 4.101] [val/perplexity_len_1024: 60.409] [val/loss_avg_len_512: 4.175] [val/perplexity_len_512: 65.014]
+[2025-10-12 09:44:43][utils:57][INFO] [P: 75.00%] [S: 1572864000/2097152000] [T: 6:35:41] [ETA: 2:11:53] [loss: 4.048] [tokens/s: 66089.555] [batches/s: 0.032] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 09:44:43][logger:171][INFO] [step: 1572864000] [train_eval/train_token_count: 1572864000] [train_eval/train_batch_count: 750] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 23741.714] [train_eval/train_update_time: 13647.155] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.061] [train_eval/perplexity_len_2048: 58.059] [train_eval/loss_avg_len_1024: 4.103] [train_eval/perplexity_len_1024: 60.523] [train_eval/loss_avg_len_512: 4.175] [train_eval/perplexity_len_512: 65.056]
+[2025-10-12 09:48:04][utils:57][INFO] [P: 76.00%] [S: 1593835520/2097152000] [T: 6:39:03] [ETA: 2:06:01] [loss: 4.010] [tokens/s: 71560.682] [batches/s: 0.034] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 09:48:04][train:194][INFO] Running validation...
+[2025-10-12 09:51:58][logger:171][INFO] [step: 1593835520] [val/train_token_count: 1593835520] [val/train_batch_count: 760] [val/train_flop_count: 0] [val/train_total_time: 23943.319] [val/train_update_time: 13848.440] [val/loss: 4.047] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 233.519] [val/val_tokens_per_second: 175403.032] [val/loss_avg_len_2048: 4.047] [val/perplexity_len_2048: 57.218] [val/loss_avg_len_1024: 4.093] [val/perplexity_len_1024: 59.908] [val/loss_avg_len_512: 4.167] [val/perplexity_len_512: 64.539]
+[2025-10-12 09:55:30][utils:57][INFO] [P: 77.00%] [S: 1614807040/2097152000] [T: 6:46:29] [ETA: 2:01:25] [loss: 4.079] [tokens/s: 65718.250] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 09:58:26][utils:57][INFO] [P: 78.00%] [S: 1635778560/2097152000] [T: 6:49:24] [ETA: 1:55:28] [loss: 4.008] [tokens/s: 72421.241] [batches/s: 0.035] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 09:58:26][train:194][INFO] Running validation...
+[2025-10-12 10:02:50][logger:171][INFO] [step: 1635778560] [val/train_token_count: 1635778560] [val/train_batch_count: 780] [val/train_flop_count: 0] [val/train_total_time: 24564.469] [val/train_update_time: 14235.557] [val/loss: 4.038] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 264.806] [val/val_tokens_per_second: 154679.153] [val/loss_avg_len_2048: 4.038] [val/perplexity_len_2048: 56.734] [val/loss_avg_len_1024: 4.085] [val/perplexity_len_1024: 59.420] [val/loss_avg_len_512: 4.160] [val/perplexity_len_512: 64.057]
+[2025-10-12 10:06:16][utils:57][INFO] [P: 79.00%] [S: 1656750080/2097152000] [T: 6:57:15] [ETA: 1:50:54] [loss: 4.032] [tokens/s: 65119.537] [batches/s: 0.031] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 10:08:46][utils:57][INFO] [P: 80.00%] [S: 1677721600/2097152000] [T: 6:59:44] [ETA: 1:44:56] [loss: 4.010] [tokens/s: 72371.598] [batches/s: 0.035] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 10:08:46][logger:171][INFO] [step: 1677721600] [train_eval/train_token_count: 1677721600] [train_eval/train_batch_count: 800] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 25184.551] [train_eval/train_update_time: 14590.464] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.039] [train_eval/perplexity_len_2048: 56.752] [train_eval/loss_avg_len_1024: 4.083] [train_eval/perplexity_len_1024: 59.312] [train_eval/loss_avg_len_512: 4.156] [train_eval/perplexity_len_512: 63.846]
+[2025-10-12 10:08:46][train:194][INFO] Running validation...
+[2025-10-12 10:11:21][logger:171][INFO] [step: 1677721600] [val/train_token_count: 1677721600] [val/train_batch_count: 800] [val/train_flop_count: 0] [val/train_total_time: 25184.551] [val/train_update_time: 14590.464] [val/loss: 4.031] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 155.413] [val/val_tokens_per_second: 263555.541] [val/loss_avg_len_2048: 4.031] [val/perplexity_len_2048: 56.323] [val/loss_avg_len_1024: 4.078] [val/perplexity_len_1024: 59.009] [val/loss_avg_len_512: 4.153] [val/perplexity_len_512: 63.642]
+[2025-10-12 10:11:21][checkpoint:111][INFO] Saving checkpoint to /workspace/forgetting-transformer/alibi_3_4_256/checkpoints/step-000001677721600.pt...
+[2025-10-12 10:11:22][checkpoint:128][INFO] Checkpoint saved to /workspace/forgetting-transformer/alibi_3_4_256/checkpoints/step-000001677721600.pt.
+[2025-10-12 10:11:22][logger:171][INFO] [step: 1677721600] [checkpoint/checkpoint_time: 0.572]
+[2025-10-12 10:12:51][utils:57][INFO] [P: 81.00%] [S: 1698693120/2097152000] [T: 7:03:49] [ETA: 1:39:25] [loss: 3.990] [tokens/s: 70026.025] [batches/s: 0.033] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 10:14:20][utils:57][INFO] [P: 82.00%] [S: 1719664640/2097152000] [T: 7:05:19] [ETA: 1:33:21] [loss: 4.000] [tokens/s: 80356.163] [batches/s: 0.038] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 10:14:20][train:194][INFO] Running validation...
+[2025-10-12 10:16:14][logger:171][INFO] [step: 1719664640] [val/train_token_count: 1719664640] [val/train_batch_count: 820] [val/train_flop_count: 0] [val/train_total_time: 25519.298] [val/train_update_time: 14768.912] [val/loss: 4.025] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 113.863] [val/val_tokens_per_second: 359729.667] [val/loss_avg_len_2048: 4.025] [val/perplexity_len_2048: 55.976] [val/loss_avg_len_1024: 4.072] [val/perplexity_len_1024: 58.661] [val/loss_avg_len_512: 4.148] [val/perplexity_len_512: 63.304]
+[2025-10-12 10:17:44][utils:57][INFO] [P: 83.00%] [S: 1740636160/2097152000] [T: 7:08:42] [ETA: 1:27:48] [loss: 4.035] [tokens/s: 79619.695] [batches/s: 0.038] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 10:19:13][utils:57][INFO] [P: 84.00%] [S: 1761607680/2097152000] [T: 7:10:11] [ETA: 1:21:56] [loss: 3.977] [tokens/s: 92070.552] [batches/s: 0.044] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 10:19:13][train:194][INFO] Running validation...
+[2025-10-12 10:21:07][logger:171][INFO] [step: 1761607680] [val/train_token_count: 1761607680] [val/train_batch_count: 840] [val/train_flop_count: 0] [val/train_total_time: 25811.901] [val/train_update_time: 14947.358] [val/loss: 4.020] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 113.577] [val/val_tokens_per_second: 360634.893] [val/loss_avg_len_2048: 4.020] [val/perplexity_len_2048: 55.708] [val/loss_avg_len_1024: 4.068] [val/perplexity_len_1024: 58.417] [val/loss_avg_len_512: 4.144] [val/perplexity_len_512: 63.068]
+[2025-10-12 10:22:36][utils:57][INFO] [P: 85.00%] [S: 1782579200/2097152000] [T: 7:13:34] [ETA: 1:16:30] [loss: 4.050] [tokens/s: 92197.578] [batches/s: 0.044] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 10:22:36][logger:171][INFO] [step: 1782579200] [train_eval/train_token_count: 1782579200] [train_eval/train_batch_count: 850] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 26014.853] [train_eval/train_update_time: 15036.595] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.014] [train_eval/perplexity_len_2048: 55.395] [train_eval/loss_avg_len_1024: 4.053] [train_eval/perplexity_len_1024: 57.594] [train_eval/loss_avg_len_512: 4.129] [train_eval/perplexity_len_512: 62.086]
+[2025-10-12 10:24:05][utils:57][INFO] [P: 86.00%] [S: 1803550720/2097152000] [T: 7:15:04] [ETA: 1:10:49] [loss: 4.033] [tokens/s: 108924.790] [batches/s: 0.052] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 10:24:05][train:194][INFO] Running validation...
+[2025-10-12 10:25:59][logger:171][INFO] [step: 1803550720] [val/train_token_count: 1803550720] [val/train_batch_count: 860] [val/train_flop_count: 0] [val/train_total_time: 26104.197] [val/train_update_time: 15125.805] [val/loss: 4.016] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 113.595] [val/val_tokens_per_second: 360578.921] [val/loss_avg_len_2048: 4.016] [val/perplexity_len_2048: 55.456] [val/loss_avg_len_1024: 4.063] [val/perplexity_len_1024: 58.157] [val/loss_avg_len_512: 4.140] [val/perplexity_len_512: 62.809]
+[2025-10-12 10:27:28][utils:57][INFO] [P: 87.00%] [S: 1824522240/2097152000] [T: 7:18:27] [ETA: 1:05:30] [loss: 3.976] [tokens/s: 109462.811] [batches/s: 0.052] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 10:28:58][utils:57][INFO] [P: 88.00%] [S: 1845493760/2097152000] [T: 7:19:56] [ETA: 0:59:59] [loss: 3.990] [tokens/s: 134287.612] [batches/s: 0.064] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 10:28:58][train:194][INFO] Running validation...
+[2025-10-12 10:30:51][logger:171][INFO] [step: 1845493760] [val/train_token_count: 1845493760] [val/train_batch_count: 880] [val/train_flop_count: 0] [val/train_total_time: 26396.507] [val/train_update_time: 15304.242] [val/loss: 4.012] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 113.629] [val/val_tokens_per_second: 360471.194] [val/loss_avg_len_2048: 4.012] [val/perplexity_len_2048: 55.277] [val/loss_avg_len_1024: 4.060] [val/perplexity_len_1024: 57.980] [val/loss_avg_len_512: 4.137] [val/perplexity_len_512: 62.635]
+[2025-10-12 10:32:21][utils:57][INFO] [P: 89.00%] [S: 1866465280/2097152000] [T: 7:23:19] [ETA: 0:54:47] [loss: 4.054] [tokens/s: 133999.162] [batches/s: 0.064] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 10:33:50][utils:57][INFO] [P: 90.00%] [S: 1887436800/2097152000] [T: 7:24:48] [ETA: 0:49:25] [loss: 3.963] [tokens/s: 155016.667] [batches/s: 0.074] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 10:33:50][logger:171][INFO] [step: 1887436800] [train_eval/train_token_count: 1887436800] [train_eval/train_batch_count: 900] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 26688.839] [train_eval/train_update_time: 15482.673] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 4.010] [train_eval/perplexity_len_2048: 55.130] [train_eval/loss_avg_len_1024: 4.053] [train_eval/perplexity_len_1024: 57.548] [train_eval/loss_avg_len_512: 4.129] [train_eval/perplexity_len_512: 62.091]
+[2025-10-12 10:33:50][train:194][INFO] Running validation...
+[2025-10-12 10:35:44][logger:171][INFO] [step: 1887436800] [val/train_token_count: 1887436800] [val/train_batch_count: 900] [val/train_flop_count: 0] [val/train_total_time: 26688.839] [val/train_update_time: 15482.673] [val/loss: 4.010] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 113.594] [val/val_tokens_per_second: 360582.024] [val/loss_avg_len_2048: 4.010] [val/perplexity_len_2048: 55.139] [val/loss_avg_len_1024: 4.058] [val/perplexity_len_1024: 57.838] [val/loss_avg_len_512: 4.135] [val/perplexity_len_512: 62.489]
+[2025-10-12 10:35:44][checkpoint:111][INFO] Saving checkpoint to /workspace/forgetting-transformer/alibi_3_4_256/checkpoints/step-000001887436800.pt...
+[2025-10-12 10:35:44][checkpoint:128][INFO] Checkpoint saved to /workspace/forgetting-transformer/alibi_3_4_256/checkpoints/step-000001887436800.pt.
+[2025-10-12 10:35:44][logger:171][INFO] [step: 1887436800] [checkpoint/checkpoint_time: 0.568]
+[2025-10-12 10:37:13][utils:57][INFO] [P: 91.00%] [S: 1908408320/2097152000] [T: 7:28:12] [ETA: 0:44:19] [loss: 4.002] [tokens/s: 142842.390] [batches/s: 0.068] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 10:38:43][utils:57][INFO] [P: 92.00%] [S: 1929379840/2097152000] [T: 7:29:41] [ETA: 0:39:06] [loss: 4.027] [tokens/s: 154985.921] [batches/s: 0.074] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 10:38:43][train:194][INFO] Running validation...
+[2025-10-12 10:40:37][logger:171][INFO] [step: 1929379840] [val/train_token_count: 1929379840] [val/train_batch_count: 920] [val/train_flop_count: 0] [val/train_total_time: 26981.709] [val/train_update_time: 15661.105] [val/loss: 4.008] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 113.745] [val/val_tokens_per_second: 360102.454] [val/loss_avg_len_2048: 4.008] [val/perplexity_len_2048: 55.042] [val/loss_avg_len_1024: 4.056] [val/perplexity_len_1024: 57.739] [val/loss_avg_len_512: 4.133] [val/perplexity_len_512: 62.391]
+[2025-10-12 10:42:06][utils:57][INFO] [P: 93.00%] [S: 1950351360/2097152000] [T: 7:33:04] [ETA: 0:34:06] [loss: 4.014] [tokens/s: 142855.467] [batches/s: 0.068] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 10:43:35][utils:57][INFO] [P: 94.00%] [S: 1971322880/2097152000] [T: 7:34:34] [ETA: 0:29:00] [loss: 3.973] [tokens/s: 154969.110] [batches/s: 0.074] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 10:43:35][train:194][INFO] Running validation...
+[2025-10-12 10:45:29][logger:171][INFO] [step: 1971322880] [val/train_token_count: 1971322880] [val/train_batch_count: 940] [val/train_flop_count: 0] [val/train_total_time: 27274.171] [val/train_update_time: 15839.539] [val/loss: 4.007] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 113.562] [val/val_tokens_per_second: 360684.215] [val/loss_avg_len_2048: 4.007] [val/perplexity_len_2048: 54.975] [val/loss_avg_len_1024: 4.055] [val/perplexity_len_1024: 57.678] [val/loss_avg_len_512: 4.132] [val/perplexity_len_512: 62.331]
+[2025-10-12 10:46:58][utils:57][INFO] [P: 95.00%] [S: 1992294400/2097152000] [T: 7:37:57] [ETA: 0:24:06] [loss: 3.987] [tokens/s: 142859.247] [batches/s: 0.068] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 10:46:58][logger:171][INFO] [step: 1992294400] [train_eval/train_token_count: 1992294400] [train_eval/train_batch_count: 950] [train_eval/train_flop_count: 0] [train_eval/train_total_time: 27477.102] [train_eval/train_update_time: 15928.765] [train_eval/window_seq_count: 51200] [train_eval/window_token_count: 104857600] [train_eval/loss_avg_len_2048: 3.998] [train_eval/perplexity_len_2048: 54.478] [train_eval/loss_avg_len_1024: 4.044] [train_eval/perplexity_len_1024: 57.054] [train_eval/loss_avg_len_512: 4.119] [train_eval/perplexity_len_512: 61.510]
+[2025-10-12 10:48:28][utils:57][INFO] [P: 96.00%] [S: 2013265920/2097152000] [T: 7:39:26] [ETA: 0:19:08] [loss: 3.987] [tokens/s: 154972.148] [batches/s: 0.074] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 10:48:28][train:194][INFO] Running validation...
+[2025-10-12 10:50:21][logger:171][INFO] [step: 2013265920] [val/train_token_count: 2013265920] [val/train_batch_count: 960] [val/train_flop_count: 0] [val/train_total_time: 27566.453] [val/train_update_time: 16017.986] [val/loss: 4.006] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 113.647] [val/val_tokens_per_second: 360415.609] [val/loss_avg_len_2048: 4.006] [val/perplexity_len_2048: 54.937] [val/loss_avg_len_1024: 4.054] [val/perplexity_len_1024: 57.638] [val/loss_avg_len_512: 4.132] [val/perplexity_len_512: 62.290]
+[2025-10-12 10:51:51][utils:57][INFO] [P: 97.00%] [S: 2034237440/2097152000] [T: 7:42:49] [ETA: 0:14:18] [loss: 4.016] [tokens/s: 142851.997] [batches/s: 0.068] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 10:53:20][utils:57][INFO] [P: 98.00%] [S: 2055208960/2097152000] [T: 7:44:18] [ETA: 0:09:28] [loss: 3.987] [tokens/s: 154968.748] [batches/s: 0.074] [MFU: 0.000] [TFLOPS: 0.000]
+[2025-10-12 10:53:20][train:194][INFO] Running validation...
+[2025-10-12 10:55:14][logger:171][INFO] [step: 2055208960] [val/train_token_count: 2055208960] [val/train_batch_count: 980] [val/train_flop_count: 0] [val/train_total_time: 27858.826] [val/train_update_time: 16196.423] [val/loss: 4.006] [val/val_token_count: 40960000] [val/val_seq_count: 20000] [val/val_time: 113.765] [val/val_tokens_per_second: 360041.670] [val/loss_avg_len_2048: 4.006] [val/perplexity_len_2048: 54.921] [val/loss_avg_len_1024: 4.054] [val/perplexity_len_1024: 57.620] [val/loss_avg_len_512: 4.132] [val/perplexity_len_512: 62.273]
+[2025-10-12 10:55:14][train:854][INFO] Training finished with 2055208960 tokens!

metrics/jsonlines/checkpoint.jsonl

@@ -0,0 +1,9 @@
+{"step": 209715200, "checkpoint/checkpoint_time": 0.564534884004388}
+{"step": 419430400, "checkpoint/checkpoint_time": 0.567023690964561}
+{"step": 629145600, "checkpoint/checkpoint_time": 0.5638952670269646}
+{"step": 838860800, "checkpoint/checkpoint_time": 0.4858379859942943}
+{"step": 1048576000, "checkpoint/checkpoint_time": 0.4735249990480952}
+{"step": 1258291200, "checkpoint/checkpoint_time": 0.5552213820046745}
+{"step": 1468006400, "checkpoint/checkpoint_time": 0.5709432679577731}
+{"step": 1677721600, "checkpoint/checkpoint_time": 0.5724008829565719}
+{"step": 1887436800, "checkpoint/checkpoint_time": 0.56779815396294}

metrics/jsonlines/model_info.jsonl

@@ -0,0 +1 @@
+{"step": 0, "model_info/total_params": 28299520, "model_info/trainable_params": 28299520, "model_info/embedding_params": 12870912, "model_info/flops_per_token": 0, "model_info/non_embedding_params": 15428608}

metrics/jsonlines/resume.jsonl

@@ -0,0 +1 @@
+{"step": 0, "resume/resume_step": 0}

metrics/jsonlines/train.jsonl

@@ -0,0 +1,98 @@
+{"step": 20971520, "train/token_count": 20971520, "train/batch_count": 10, "train/flop_count": 0, "train/total_time": 219.44772298802854, "train/update_time": 219.17982283898164, "train/lr": 0.0009000000000000001, "train/loss": 9.77186393737793, "train/global_grad_norm": 1.2174378633499146}
+{"step": 41943040, "train/token_count": 41943040, "train/batch_count": 20, "train/flop_count": 0, "train/total_time": 393.3033275610069, "train/update_time": 392.7191811740049, "train/lr": 0.0009997960964140947, "train/loss": 8.123015403747559, "train/global_grad_norm": 0.9983794093132019}
+{"step": 62914560, "train/token_count": 62914560, "train/batch_count": 30, "train/flop_count": 0, "train/total_time": 863.6309026500094, "train/update_time": 595.3849820840405, "train/lr": 0.0009990914580222257, "train/loss": 7.504764556884766, "train/global_grad_norm": 0.5670243501663208}
+{"step": 83886080, "train/token_count": 83886080, "train/batch_count": 40, "train/flop_count": 0, "train/total_time": 1014.3213793600444, "train/update_time": 745.8892338059959, "train/lr": 0.0009978842768382998, "train/loss": 7.151111602783203, "train/global_grad_norm": 0.44631311297416687}
+{"step": 104857600, "train/token_count": 104857600, "train/batch_count": 50, "train/flop_count": 0, "train/total_time": 1483.1021124350373, "train/update_time": 908.4019459009869, "train/lr": 0.0009961757683914405, "train/loss": 6.869814872741699, "train/global_grad_norm": 0.3835476338863373}
+{"step": 125829120, "train/token_count": 125829120, "train/batch_count": 60, "train/flop_count": 0, "train/total_time": 1633.8735543090152, "train/update_time": 1058.99380204099, "train/lr": 0.00099396765300483, "train/loss": 6.582631587982178, "train/global_grad_norm": 0.342731237411499}
+{"step": 146800640, "train/token_count": 146800640, "train/batch_count": 70, "train/flop_count": 0, "train/total_time": 2102.5612953370437, "train/update_time": 1209.294771298999, "train/lr": 0.0009912621540634887, "train/loss": 6.3764543533325195, "train/global_grad_norm": 0.31251585483551025}
+{"step": 167772160, "train/token_count": 167772160, "train/batch_count": 80, "train/flop_count": 0, "train/total_time": 2296.034582130029, "train/update_time": 1402.5742697739624, "train/lr": 0.000988061995775515, "train/loss": 6.187304496765137, "train/global_grad_norm": 0.457292377948761}
+{"step": 188743680, "train/token_count": 188743680, "train/batch_count": 90, "train/flop_count": 0, "train/total_time": 2738.4253872900154, "train/update_time": 1568.2938627917902, "train/lr": 0.0009843704004290394, "train/loss": 6.000740051269531, "train/global_grad_norm": 0.45764341950416565}
+{"step": 209715200, "train/token_count": 209715200, "train/batch_count": 100, "train/flop_count": 0, "train/total_time": 2950.672681943048, "train/update_time": 1780.2358716417802, "train/lr": 0.0009801910851476522, "train/loss": 5.888554573059082, "train/global_grad_norm": 0.4506590664386749}
+{"step": 230686720, "train/token_count": 230686720, "train/batch_count": 110, "train/flop_count": 0, "train/total_time": 3388.6962326810462, "train/update_time": 1974.5968189326813, "train/lr": 0.0009755282581475768, "train/loss": 5.764591693878174, "train/global_grad_norm": 0.38608840107917786}
+{"step": 251658240, "train/token_count": 251658240, "train/batch_count": 120, "train/flop_count": 0, "train/total_time": 3595.7228351200465, "train/update_time": 2181.321583448793, "train/lr": 0.0009703866145003512, "train/loss": 5.629734039306641, "train/global_grad_norm": 0.6687659025192261}
+{"step": 272629760, "train/token_count": 272629760, "train/batch_count": 130, "train/flop_count": 0, "train/total_time": 4033.824217539048, "train/update_time": 2393.338223681785, "train/lr": 0.0009647713314052896, "train/loss": 5.562553882598877, "train/global_grad_norm": 0.46099352836608887}
+{"step": 293601280, "train/token_count": 293601280, "train/batch_count": 140, "train/flop_count": 0, "train/total_time": 4217.221132909006, "train/update_time": 2576.439015968761, "train/lr": 0.0009586880629764817, "train/loss": 5.487265586853027, "train/global_grad_norm": 0.7035791873931885}
+{"step": 314572800, "train/token_count": 314572800, "train/batch_count": 150, "train/flop_count": 0, "train/total_time": 4684.9639007470105, "train/update_time": 2788.3531909267767, "train/lr": 0.0009521429345495787, "train/loss": 5.383974075317383, "train/global_grad_norm": 0.5269623398780823}
+{"step": 335544320, "train/token_count": 335544320, "train/batch_count": 160, "train/flop_count": 0, "train/total_time": 4838.394258918008, "train/update_time": 2941.477261595719, "train/lr": 0.0009451425365140996, "train/loss": 5.3296051025390625, "train/global_grad_norm": 0.3451927602291107}
+{"step": 356515840, "train/token_count": 356515840, "train/batch_count": 170, "train/flop_count": 0, "train/total_time": 5307.577416299027, "train/update_time": 3118.180899302766, "train/lr": 0.000937693917677468, "train/loss": 5.232334136962891, "train/global_grad_norm": 0.49183663725852966}
+{"step": 377487360, "train/token_count": 377487360, "train/batch_count": 180, "train/flop_count": 0, "train/total_time": 5457.476125103014, "train/update_time": 3267.897231984767, "train/lr": 0.0009298045781674596, "train/loss": 5.203054904937744, "train/global_grad_norm": 0.3954695463180542}
+{"step": 398458880, "train/token_count": 398458880, "train/batch_count": 190, "train/flop_count": 0, "train/total_time": 5926.165717172029, "train/update_time": 3418.016206740809, "train/lr": 0.0009214824618802108, "train/loss": 5.189924716949463, "train/global_grad_norm": 0.7796792984008789}
+{"step": 419430400, "train/token_count": 419430400, "train/batch_count": 200, "train/flop_count": 0, "train/total_time": 6104.984292554029, "train/update_time": 3596.637184852676, "train/lr": 0.000912735948481387, "train/loss": 5.086824893951416, "train/global_grad_norm": 0.4062754809856415}
+{"step": 440401920, "train/token_count": 440401920, "train/batch_count": 210, "train/flop_count": 0, "train/total_time": 6550.405725484015, "train/update_time": 3750.722188989632, "train/lr": 0.0009035738449685707, "train/loss": 5.0450758934021, "train/global_grad_norm": 0.5696613788604736}
+{"step": 461373440, "train/token_count": 461373440, "train/batch_count": 220, "train/flop_count": 0, "train/total_time": 6762.651720013004, "train/update_time": 3962.670169218676, "train/lr": 0.0008940053768033609, "train/loss": 5.003251075744629, "train/global_grad_norm": 0.4241951107978821}
+{"step": 482344960, "train/token_count": 482344960, "train/batch_count": 230, "train/flop_count": 0, "train/total_time": 7201.653420041024, "train/update_time": 4147.277713002753, "train/lr": 0.0008840401786221159, "train/loss": 4.949978828430176, "train/global_grad_norm": 0.48395395278930664}
+{"step": 503316480, "train/token_count": 503316480, "train/batch_count": 240, "train/flop_count": 0, "train/total_time": 7413.931084726995, "train/update_time": 4359.249880091811, "train/lr": 0.0008736882845346905, "train/loss": 4.89894962310791, "train/global_grad_norm": 0.5247655510902405}
+{"step": 524288000, "train/token_count": 524288000, "train/batch_count": 250, "train/flop_count": 0, "train/total_time": 7847.669167418033, "train/update_time": 4567.680281858717, "train/lr": 0.0008629601180209381, "train/loss": 4.89341926574707, "train/global_grad_norm": 0.5477429032325745}
+{"step": 545259520, "train/token_count": 545259520, "train/batch_count": 260, "train/flop_count": 0, "train/total_time": 8040.308323490026, "train/update_time": 4760.0058897937415, "train/lr": 0.0008518664814351503, "train/loss": 4.842683792114258, "train/global_grad_norm": 0.5116428732872009}
+{"step": 566231040, "train/token_count": 566231040, "train/batch_count": 270, "train/flop_count": 0, "train/total_time": 8496.36842228804, "train/update_time": 4972.021029465657, "train/lr": 0.0008404185451290017, "train/loss": 4.826287746429443, "train/global_grad_norm": 0.5284518599510193}
+{"step": 587202560, "train/token_count": 587202560, "train/batch_count": 280, "train/flop_count": 0, "train/total_time": 8661.466409554007, "train/update_time": 5136.805537326669, "train/lr": 0.0008286278362039527, "train/loss": 4.773619174957275, "train/global_grad_norm": 0.5465279221534729}
+{"step": 608174080, "train/token_count": 608174080, "train/batch_count": 290, "train/flop_count": 0, "train/total_time": 9131.361868123, "train/update_time": 5328.725031249633, "train/lr": 0.0008165062269044352, "train/loss": 4.738215446472168, "train/global_grad_norm": 0.43803009390830994}
+{"step": 629145600, "train/token_count": 629145600, "train/batch_count": 300, "train/flop_count": 0, "train/total_time": 9281.588875080051, "train/update_time": 5478.766576015565, "train/lr": 0.0008040659226635089, "train/loss": 4.710612773895264, "train/global_grad_norm": 0.5540741086006165}
+{"step": 650117120, "train/token_count": 650117120, "train/batch_count": 310, "train/flop_count": 0, "train/total_time": 9750.55400532001, "train/update_time": 5628.910955499508, "train/lr": 0.0007913194498130252, "train/loss": 4.7304229736328125, "train/global_grad_norm": 0.45888814330101013}
+{"step": 671088640, "train/token_count": 671088640, "train/batch_count": 320, "train/flop_count": 0, "train/total_time": 9914.599941182008, "train/update_time": 5792.780345006497, "train/lr": 0.000778279642970672, "train/loss": 4.656280994415283, "train/global_grad_norm": 0.4830974340438843}
+{"step": 692060160, "train/token_count": 692060160, "train/batch_count": 330, "train/flop_count": 0, "train/total_time": 10369.58782212704, "train/update_time": 5942.496549489384, "train/lr": 0.0007649596321166025, "train/loss": 4.677618026733398, "train/global_grad_norm": 0.7399375438690186}
+{"step": 713031680, "train/token_count": 713031680, "train/batch_count": 340, "train/flop_count": 0, "train/total_time": 10574.152368383016, "train/update_time": 6146.868059828412, "train/lr": 0.0007513728293726579, "train/loss": 4.6279706954956055, "train/global_grad_norm": 0.4657606780529022}
+{"step": 734003200, "train/token_count": 734003200, "train/batch_count": 350, "train/flop_count": 0, "train/total_time": 11014.551226956013, "train/update_time": 6321.31326986436, "train/lr": 0.0007375329154974975, "train/loss": 4.613265037536621, "train/global_grad_norm": 0.7494572997093201}
+{"step": 754974720, "train/token_count": 754974720, "train/batch_count": 360, "train/flop_count": 0, "train/total_time": 11226.902240763011, "train/update_time": 6533.353484157473, "train/lr": 0.0007234538261112341, "train/loss": 4.532633304595947, "train/global_grad_norm": 0.5424163937568665}
+{"step": 775946240, "train/token_count": 775946240, "train/batch_count": 370, "train/flop_count": 0, "train/total_time": 11661.993830877997, "train/update_time": 6734.094668152509, "train/lr": 0.0007091497376634464, "train/loss": 4.551605224609375, "train/global_grad_norm": 0.5107011795043945}
+{"step": 796917760, "train/token_count": 796917760, "train/batch_count": 380, "train/flop_count": 0, "train/total_time": 11862.473320519028, "train/update_time": 6934.267906721507, "train/lr": 0.0006946350531586958, "train/loss": 4.527378559112549, "train/global_grad_norm": 0.5982224941253662}
+{"step": 817889280, "train/token_count": 817889280, "train/batch_count": 390, "train/flop_count": 0, "train/total_time": 12308.825131079007, "train/update_time": 7146.272098292538, "train/lr": 0.0006799243876539214, "train/loss": 4.523767948150635, "train/global_grad_norm": 0.4483889043331146}
+{"step": 838860800, "train/token_count": 838860800, "train/batch_count": 400, "train/flop_count": 0, "train/total_time": 12483.889439035032, "train/update_time": 7321.039978560642, "train/lr": 0.0006650325535423166, "train/loss": 4.428459644317627, "train/global_grad_norm": 0.6677308678627014}
+{"step": 859832320, "train/token_count": 859832320, "train/batch_count": 410, "train/flop_count": 0, "train/total_time": 12954.41088695702, "train/update_time": 7525.117961935524, "train/lr": 0.0006499745456385053, "train/loss": 4.447395324707031, "train/global_grad_norm": 0.5507714152336121}
+{"step": 880803840, "train/token_count": 880803840, "train/batch_count": 420, "train/flop_count": 0, "train/total_time": 13104.464458574017, "train/update_time": 7674.994709033519, "train/lr": 0.0006347655260800339, "train/loss": 4.442117214202881, "train/global_grad_norm": 0.5809928774833679}
+{"step": 901775360, "train/token_count": 901775360, "train/batch_count": 430, "train/flop_count": 0, "train/total_time": 13572.86144918704, "train/update_time": 7837.980615513574, "train/lr": 0.0006194208090603844, "train/loss": 4.436192989349365, "train/global_grad_norm": 0.5325289368629456}
+{"step": 922746880, "train/token_count": 922746880, "train/batch_count": 440, "train/flop_count": 0, "train/total_time": 13721.99477496103, "train/update_time": 7986.941047978529, "train/lr": 0.0006039558454088796, "train/loss": 4.456014633178711, "train/global_grad_norm": 0.6674598455429077}
+{"step": 943718400, "train/token_count": 943718400, "train/batch_count": 450, "train/flop_count": 0, "train/total_time": 14189.86244938703, "train/update_time": 8136.602942996542, "train/lr": 0.0005883862070330078, "train/loss": 4.393319606781006, "train/global_grad_norm": 0.5084896683692932}
+{"step": 964689920, "train/token_count": 964689920, "train/batch_count": 460, "train/flop_count": 0, "train/total_time": 14381.773025262053, "train/update_time": 8328.335106454615, "train/lr": 0.0005727275712388317, "train/loss": 4.356149673461914, "train/global_grad_norm": 0.5596035718917847}
+{"step": 985661440, "train/token_count": 985661440, "train/batch_count": 470, "train/flop_count": 0, "train/total_time": 14823.617820873042, "train/update_time": 8492.146036952618, "train/lr": 0.0005569957049452703, "train/loss": 4.362613677978516, "train/global_grad_norm": 0.5118389129638672}
+{"step": 1006632960, "train/token_count": 1006632960, "train/batch_count": 480, "train/flop_count": 0, "train/total_time": 15035.86255901301, "train/update_time": 8704.087580051564, "train/lr": 0.0005412064488081482, "train/loss": 4.340407371520996, "train/global_grad_norm": 0.5723304152488708}
+{"step": 1027604480, "train/token_count": 1027604480, "train/batch_count": 490, "train/flop_count": 0, "train/total_time": 15473.11414746102, "train/update_time": 8896.564982390497, "train/lr": 0.0005253757012699972, "train/loss": 4.335324287414551, "train/global_grad_norm": 0.5161768198013306}
+{"step": 1048576000, "train/token_count": 1048576000, "train/batch_count": 500, "train/flop_count": 0, "train/total_time": 15681.829017209006, "train/update_time": 9104.969846359396, "train/lr": 0.0005095194025516734, "train/loss": 4.303962707519531, "train/global_grad_norm": 0.5649275779724121}
+{"step": 1069547520, "train/token_count": 1069547520, "train/batch_count": 510, "train/flop_count": 0, "train/total_time": 16119.577212364005, "train/update_time": 9316.950864796352, "train/lr": 0.0004936535186019053, "train/loss": 4.29068660736084, "train/global_grad_norm": 0.610883355140686}
+{"step": 1090519040, "train/token_count": 1090519040, "train/batch_count": 520, "train/flop_count": 0, "train/total_time": 16303.791181104025, "train/update_time": 9500.859303092468, "train/lr": 0.00047779402502093696, "train/loss": 4.2827653884887695, "train/global_grad_norm": 0.5993229150772095}
+{"step": 1111490560, "train/token_count": 1111490560, "train/batch_count": 530, "train/flop_count": 0, "train/total_time": 16770.228860588046, "train/update_time": 9712.85516443639, "train/lr": 0.0004619568909744525, "train/loss": 4.242770671844482, "train/global_grad_norm": 0.49561238288879395}
+{"step": 1132462080, "train/token_count": 1132462080, "train/batch_count": 540, "train/flop_count": 0, "train/total_time": 16924.293944418023, "train/update_time": 9866.63232135144, "train/lr": 0.00044615806311398067, "train/loss": 4.251742839813232, "train/global_grad_norm": 0.6209728717803955}
+{"step": 1153433600, "train/token_count": 1153433600, "train/batch_count": 550, "train/flop_count": 0, "train/total_time": 17393.370531369, "train/update_time": 10044.968985380197, "train/lr": 0.0004304134495199673, "train/loss": 4.189674377441406, "train/global_grad_norm": 0.5244336128234863}
+{"step": 1174405120, "train/token_count": 1174405120, "train/batch_count": 560, "train/flop_count": 0, "train/total_time": 17543.088614015025, "train/update_time": 10194.5136101502, "train/lr": 0.0004147389036836882, "train/loss": 4.233956813812256, "train/global_grad_norm": 0.5448895692825317}
+{"step": 1195376640, "train/token_count": 1195376640, "train/batch_count": 570, "train/flop_count": 0, "train/total_time": 18010.892701787, "train/update_time": 10344.159968029184, "train/lr": 0.0003991502085441259, "train/loss": 4.165826797485352, "train/global_grad_norm": 0.6113353967666626}
+{"step": 1216348160, "train/token_count": 1216348160, "train/batch_count": 580, "train/flop_count": 0, "train/total_time": 18186.98511047603, "train/update_time": 10520.074710074114, "train/lr": 0.0003836630605958888, "train/loss": 4.22281551361084, "train/global_grad_norm": 0.5304960012435913}
+{"step": 1237319680, "train/token_count": 1237319680, "train/batch_count": 590, "train/flop_count": 0, "train/total_time": 18632.073160929023, "train/update_time": 10671.874259623233, "train/lr": 0.00036829305408417155, "train/loss": 4.211955547332764, "train/global_grad_norm": 0.5258879065513611}
+{"step": 1258291200, "train/token_count": 1258291200, "train/batch_count": 600, "train/flop_count": 0, "train/total_time": 18844.378502471023, "train/update_time": 10883.880087946192, "train/lr": 0.000353055665302672, "train/loss": 4.207913875579834, "train/global_grad_norm": 0.6525558829307556}
+{"step": 1279262720, "train/token_count": 1279262720, "train/batch_count": 610, "train/flop_count": 0, "train/total_time": 19284.370175764023, "train/update_time": 11066.854351600225, "train/lr": 0.0003379662370102746, "train/loss": 4.168120861053467, "train/global_grad_norm": 0.5861026644706726}
+{"step": 1300234240, "train/token_count": 1300234240, "train/batch_count": 620, "train/flop_count": 0, "train/total_time": 19496.670543703018, "train/update_time": 11278.836069991172, "train/lr": 0.00032303996298219405, "train/loss": 4.139932632446289, "train/global_grad_norm": 0.5590474605560303}
+{"step": 1321205760, "train/token_count": 1321205760, "train/batch_count": 630, "train/flop_count": 0, "train/total_time": 19930.58125682705, "train/update_time": 11486.317253635323, "train/lr": 0.00030829187271113034, "train/loss": 4.151315689086914, "train/global_grad_norm": 0.48951366543769836}
+{"step": 1342177280, "train/token_count": 1342177280, "train/batch_count": 640, "train/flop_count": 0, "train/total_time": 20124.23348677403, "train/update_time": 11679.664113470295, "train/lr": 0.0002937368162738445, "train/loss": 4.140133857727051, "train/global_grad_norm": 0.5501073002815247}
+{"step": 1363148800, "train/token_count": 1363148800, "train/batch_count": 650, "train/flop_count": 0, "train/total_time": 20578.88107217505, "train/update_time": 11891.677284248348, "train/lr": 0.0002793894493783894, "train/loss": 4.110686302185059, "train/global_grad_norm": 0.859492838382721}
+{"step": 1384120320, "train/token_count": 1384120320, "train/batch_count": 660, "train/flop_count": 0, "train/total_time": 20744.93009287404, "train/update_time": 12057.427525710431, "train/lr": 0.00026526421860705474, "train/loss": 4.138786792755127, "train/global_grad_norm": 0.5685120224952698}
+{"step": 1405091840, "train/token_count": 1405091840, "train/batch_count": 670, "train/flop_count": 0, "train/total_time": 21214.90017826401, "train/update_time": 12251.372320065391, "train/lr": 0.0002513753468698824, "train/loss": 4.0760040283203125, "train/global_grad_norm": 0.5391287207603455}
+{"step": 1426063360, "train/token_count": 1426063360, "train/batch_count": 680, "train/flop_count": 0, "train/total_time": 21364.400057210005, "train/update_time": 12400.716242285387, "train/lr": 0.00023773681908340283, "train/loss": 4.092895984649658, "train/global_grad_norm": 0.5448238849639893}
+{"step": 1447034880, "train/token_count": 1447034880, "train/batch_count": 690, "train/flop_count": 0, "train/total_time": 21831.916811336007, "train/update_time": 12550.552763604384, "train/lr": 0.00022436236808900823, "train/loss": 4.098409652709961, "train/global_grad_norm": 0.5497803092002869}
+{"step": 1468006400, "train/token_count": 1468006400, "train/batch_count": 700, "train/flop_count": 0, "train/total_time": 21992.386447268014, "train/update_time": 12710.86209141434, "train/lr": 0.00021126546082514682, "train/loss": 4.090141773223877, "train/global_grad_norm": 0.4459550082683563}
+{"step": 1488977920, "train/token_count": 1488977920, "train/batch_count": 710, "train/flop_count": 0, "train/total_time": 22450.115988292033, "train/update_time": 12860.017111644556, "train/lr": 0.00019845928476725522, "train/loss": 4.094099044799805, "train/global_grad_norm": 0.48487746715545654}
+{"step": 1509949440, "train/token_count": 1509949440, "train/batch_count": 720, "train/flop_count": 0, "train/total_time": 22653.044829952996, "train/update_time": 13062.762519414595, "train/lr": 0.0001859567346490913, "train/loss": 4.064176082611084, "train/global_grad_norm": 0.6045382022857666}
+{"step": 1530920960, "train/token_count": 1530920960, "train/batch_count": 730, "train/flop_count": 0, "train/total_time": 23093.526105974044, "train/update_time": 13235.4935254085, "train/lr": 0.00017377039947882782, "train/loss": 4.084225177764893, "train/global_grad_norm": 0.42127346992492676}
+{"step": 1551892480, "train/token_count": 1551892480, "train/batch_count": 740, "train/flop_count": 0, "train/total_time": 23305.888173635, "train/update_time": 13447.544946793583, "train/lr": 0.00016191254986299043, "train/loss": 4.059417724609375, "train/global_grad_norm": 0.43410712480545044}
+{"step": 1572864000, "train/token_count": 1572864000, "train/batch_count": 750, "train/flop_count": 0, "train/total_time": 23741.714356134005, "train/update_time": 13647.155335442629, "train/lr": 0.00015039512565099468, "train/loss": 4.048225402832031, "train/global_grad_norm": 0.4433448016643524}
+{"step": 1593835520, "train/token_count": 1593835520, "train/batch_count": 760, "train/flop_count": 0, "train/total_time": 23943.319328956015, "train/update_time": 13848.439878219564, "train/lr": 0.00013922972391273224, "train/loss": 4.010053634643555, "train/global_grad_norm": 0.5291637182235718}
+{"step": 1614807040, "train/token_count": 1614807040, "train/batch_count": 770, "train/flop_count": 0, "train/total_time": 24389.08542349504, "train/update_time": 14060.475522507564, "train/lr": 0.00012842758726130281, "train/loss": 4.079256534576416, "train/global_grad_norm": 0.39220452308654785}
+{"step": 1635778560, "train/token_count": 1635778560, "train/batch_count": 780, "train/flop_count": 0, "train/total_time": 24564.46875228104, "train/update_time": 14235.556923599623, "train/lr": 0.00011799959253265679, "train/loss": 4.007717609405518, "train/global_grad_norm": 0.3393424153327942}
+{"step": 1656750080, "train/token_count": 1656750080, "train/batch_count": 790, "train/flop_count": 0, "train/total_time": 25035.12217405904, "train/update_time": 14441.209321577568, "train/lr": 0.00010795623983354214, "train/loss": 4.031998634338379, "train/global_grad_norm": 0.47528231143951416}
+{"step": 1677721600, "train/token_count": 1677721600, "train/batch_count": 800, "train/flop_count": 0, "train/total_time": 25184.550976835017, "train/update_time": 14590.463811040623, "train/lr": 9.830764196878872e-05, "train/loss": 4.010214328765869, "train/global_grad_norm": 0.38191747665405273}
+{"step": 1698693120, "train/token_count": 1698693120, "train/batch_count": 810, "train/flop_count": 0, "train/total_time": 25429.942887034034, "train/update_time": 14679.69327192055, "train/lr": 8.906351425856951e-05, "train/loss": 3.989518642425537, "train/global_grad_norm": 0.3788834810256958}
+{"step": 1719664640, "train/token_count": 1719664640, "train/batch_count": 820, "train/flop_count": 0, "train/total_time": 25519.29751138203, "train/update_time": 14768.911977301526, "train/lr": 8.02331647558977e-05, "train/loss": 3.9995009899139404, "train/global_grad_norm": 0.37431642413139343}
+{"step": 1740636160, "train/token_count": 1740636160, "train/batch_count": 830, "train/flop_count": 0, "train/total_time": 25722.537228438014, "train/update_time": 14858.13312200259, "train/lr": 7.182548487420554e-05, "train/loss": 4.0348801612854, "train/global_grad_norm": 0.39566728472709656}
+{"step": 1761607680, "train/token_count": 1761607680, "train/batch_count": 840, "train/flop_count": 0, "train/total_time": 25811.900586274045, "train/update_time": 14947.35764870455, "train/lr": 6.384894043444556e-05, "train/loss": 3.9766266345977783, "train/global_grad_norm": 0.40857023000717163}
+{"step": 1782579200, "train/token_count": 1782579200, "train/batch_count": 850, "train/flop_count": 0, "train/total_time": 26014.853294235014, "train/update_time": 15036.594645244593, "train/lr": 5.6311563140726166e-05, "train/loss": 4.0503740310668945, "train/global_grad_norm": 0.33130380511283875}
+{"step": 1803550720, "train/token_count": 1803550720, "train/batch_count": 860, "train/flop_count": 0, "train/total_time": 26104.19660033105, "train/update_time": 15125.80504047859, "train/lr": 4.922094249306547e-05, "train/loss": 4.03305196762085, "train/global_grad_norm": 0.3090103566646576}
+{"step": 1824522240, "train/token_count": 1824522240, "train/batch_count": 870, "train/flop_count": 0, "train/total_time": 26307.15804652701, "train/update_time": 15215.028811053548, "train/lr": 4.2584218145409916e-05, "train/loss": 3.975665807723999, "train/global_grad_norm": 0.2927410304546356}
+{"step": 1845493760, "train/token_count": 1845493760, "train/batch_count": 880, "train/flop_count": 0, "train/total_time": 26396.506954086013, "train/update_time": 15304.24179791665, "train/lr": 3.6408072716606236e-05, "train/loss": 3.989638090133667, "train/global_grad_norm": 0.2772652506828308}
+{"step": 1866465280, "train/token_count": 1866465280, "train/batch_count": 890, "train/flop_count": 0, "train/total_time": 26599.493567029014, "train/update_time": 15393.460775722808, "train/lr": 3.069872506157217e-05, "train/loss": 4.053800582885742, "train/global_grad_norm": 0.27483704686164856}
+{"step": 1887436800, "train/token_count": 1887436800, "train/batch_count": 900, "train/flop_count": 0, "train/total_time": 26688.83935572201, "train/update_time": 15482.672627257823, "train/lr": 2.5461924009435368e-05, "train/loss": 3.9634201526641846, "train/global_grad_norm": 0.2686466574668884}
+{"step": 1908408320, "train/token_count": 1908408320, "train/batch_count": 910, "train/flop_count": 0, "train/total_time": 26892.36250021303, "train/update_time": 15571.891412241792, "train/lr": 2.0702942574950812e-05, "train/loss": 4.001792907714844, "train/global_grad_norm": 0.2545761168003082}
+{"step": 1929379840, "train/token_count": 1929379840, "train/batch_count": 920, "train/flop_count": 0, "train/total_time": 26981.709376943007, "train/update_time": 15661.105290838808, "train/lr": 1.642657264902142e-05, "train/loss": 4.026549816131592, "train/global_grad_norm": 0.2649659812450409}
+{"step": 1950351360, "train/token_count": 1950351360, "train/batch_count": 930, "train/flop_count": 0, "train/total_time": 27184.824638435035, "train/update_time": 15750.32617144374, "train/lr": 1.2637120173670358e-05, "train/loss": 4.013813495635986, "train/global_grad_norm": 0.2429317831993103}
+{"step": 1971322880, "train/token_count": 1971322880, "train/batch_count": 940, "train/flop_count": 0, "train/total_time": 27274.170978915005, "train/update_time": 15839.539326993749, "train/lr": 9.338400806321978e-06, "train/loss": 3.972702741622925, "train/global_grad_norm": 0.2406640350818634}
+{"step": 1992294400, "train/token_count": 1992294400, "train/batch_count": 950, "train/flop_count": 0, "train/total_time": 27477.10213337402, "train/update_time": 15928.765013526718, "train/lr": 6.533736077758867e-06, "train/loss": 3.987436532974243, "train/global_grad_norm": 0.23872317373752594}
+{"step": 2013265920, "train/token_count": 2013265920, "train/batch_count": 960, "train/flop_count": 0, "train/total_time": 27566.45336975, "train/update_time": 16017.985979812802, "train/lr": 4.2259500476214406e-06, "train/loss": 3.987391471862793, "train/global_grad_norm": 0.2312818020582199}
+{"step": 2034237440, "train/token_count": 2034237440, "train/batch_count": 970, "train/flop_count": 0, "train/total_time": 27769.478837262024, "train/update_time": 16107.212218919827, "train/lr": 2.417366460819359e-06, "train/loss": 4.015570640563965, "train/global_grad_norm": 0.22287066280841827}
+{"step": 2055208960, "train/token_count": 2055208960, "train/batch_count": 980, "train/flop_count": 0, "train/total_time": 27858.825974697014, "train/update_time": 16196.423335834814, "train/lr": 1.1098064077174619e-06, "train/loss": 3.98713755607605, "train/global_grad_norm": 0.21902944147586823}

metrics/jsonlines/train_data_info.jsonl

@@ -0,0 +1 @@
+{"step": 0, "train_data_info/vocab_size": 50277, "train_data_info/global_tokens_per_batch": 2097152, "train_data_info/local_tokens_per_batch": 2097152, "train_data_info/batch_len": 2048, "train_data_info/seq_len": 2048, "train_data_info/total_tokens": 2055208960, "train_data_info/global_batch_size": 1024, "train_data_info/local_batch_size": 1024}

metrics/jsonlines/train_eval.jsonl

@@ -0,0 +1,19 @@
+{"step": 104857600, "train_eval/train_token_count": 104857600, "train_eval/train_batch_count": 50, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 1483.1021124350373, "train_eval/train_update_time": 908.4019459009869, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 8.24263097376177, "train_eval/perplexity_len_2048": 3799.523616442905, "train_eval/loss_avg_len_1024": 8.243660057830784, "train_eval/perplexity_len_1024": 3803.4356582311275, "train_eval/loss_avg_len_512": 8.244735129414474, "train_eval/perplexity_len_512": 3807.526822580914}
+{"step": 209715200, "train_eval/train_token_count": 209715200, "train_eval/train_batch_count": 100, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 2950.672681943048, "train_eval/train_update_time": 1780.2358716417802, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 6.304204806905582, "train_eval/perplexity_len_2048": 546.866550733703, "train_eval/loss_avg_len_1024": 6.30830627080104, "train_eval/perplexity_len_1024": 549.1141101383427, "train_eval/loss_avg_len_512": 6.314299033218994, "train_eval/perplexity_len_512": 552.4147004897641}
+{"step": 314572800, "train_eval/train_token_count": 314572800, "train_eval/train_batch_count": 150, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 4684.9639007470105, "train_eval/train_update_time": 2788.3531909267767, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 5.612906693891327, "train_eval/perplexity_len_2048": 273.93933969161543, "train_eval/loss_avg_len_1024": 5.618678117767923, "train_eval/perplexity_len_1024": 275.52493089567076, "train_eval/loss_avg_len_512": 5.63033959157925, "train_eval/perplexity_len_512": 278.75676501146086}
+{"step": 419430400, "train_eval/train_token_count": 419430400, "train_eval/train_batch_count": 200, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 6104.984292554029, "train_eval/train_update_time": 3596.637184852676, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 5.231335072982001, "train_eval/perplexity_len_2048": 187.04235209109083, "train_eval/loss_avg_len_1024": 5.240096947866332, "train_eval/perplexity_len_1024": 188.68839445620821, "train_eval/loss_avg_len_512": 5.255696088007826, "train_eval/perplexity_len_512": 191.65484807445753}
+{"step": 524288000, "train_eval/train_token_count": 524288000, "train_eval/train_batch_count": 250, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 7847.669167418033, "train_eval/train_update_time": 4567.680281858717, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.980712333356914, "train_eval/perplexity_len_2048": 145.57804482522243, "train_eval/loss_avg_len_1024": 4.989459048826356, "train_eval/perplexity_len_1024": 146.85695955994623, "train_eval/loss_avg_len_512": 5.007807282713766, "train_eval/perplexity_len_512": 149.5763975512185}
+{"step": 629145600, "train_eval/train_token_count": 629145600, "train_eval/train_batch_count": 300, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 9281.588875080051, "train_eval/train_update_time": 5478.766576015565, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.805959661819288, "train_eval/perplexity_len_2048": 122.23674068512044, "train_eval/loss_avg_len_1024": 4.8155524545809385, "train_eval/perplexity_len_1024": 123.41497464824776, "train_eval/loss_avg_len_512": 4.837010983348955, "train_eval/perplexity_len_512": 126.0918971220228}
+{"step": 734003200, "train_eval/train_token_count": 734003200, "train_eval/train_batch_count": 350, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 11014.551226956013, "train_eval/train_update_time": 6321.31326986436, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.663479676307288, "train_eval/perplexity_len_2048": 106.00430179998108, "train_eval/loss_avg_len_1024": 4.677575560776713, "train_eval/perplexity_len_1024": 107.50910705570554, "train_eval/loss_avg_len_512": 4.7043979177456645, "train_eval/perplexity_len_512": 110.43177591493382}
+{"step": 838860800, "train_eval/train_token_count": 838860800, "train_eval/train_batch_count": 400, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 12483.889439035032, "train_eval/train_update_time": 7321.039978560642, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.545382353753193, "train_eval/perplexity_len_2048": 94.19643668589313, "train_eval/loss_avg_len_1024": 4.5609763441320315, "train_eval/perplexity_len_1024": 95.67684777192169, "train_eval/loss_avg_len_512": 4.592948676595988, "train_eval/perplexity_len_512": 98.78528699362364}
+{"step": 943718400, "train_eval/train_token_count": 943718400, "train_eval/train_batch_count": 450, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 14189.86244938703, "train_eval/train_update_time": 8136.602942996542, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.442484366352373, "train_eval/perplexity_len_2048": 84.98581552193247, "train_eval/loss_avg_len_1024": 4.464405859036887, "train_eval/perplexity_len_1024": 86.86940153422208, "train_eval/loss_avg_len_512": 4.504311476942166, "train_eval/perplexity_len_512": 90.40607594695932}
+{"step": 1048576000, "train_eval/train_token_count": 1048576000, "train_eval/train_batch_count": 500, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 15681.829017209006, "train_eval/train_update_time": 9104.969846359396, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.349756015720322, "train_eval/perplexity_len_2048": 77.45956170418766, "train_eval/loss_avg_len_1024": 4.3738111120650505, "train_eval/perplexity_len_1024": 79.34545060552118, "train_eval/loss_avg_len_512": 4.4205777487907465, "train_eval/perplexity_len_512": 83.14430800790313}
+{"step": 1153433600, "train_eval/train_token_count": 1153433600, "train_eval/train_batch_count": 550, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 17393.370531369, "train_eval/train_update_time": 10044.968985380197, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.263560818342057, "train_eval/perplexity_len_2048": 71.06257439048836, "train_eval/loss_avg_len_1024": 4.29050152948963, "train_eval/perplexity_len_1024": 73.00307251356847, "train_eval/loss_avg_len_512": 4.341866313039864, "train_eval/perplexity_len_512": 76.85083329331437}
+{"step": 1258291200, "train_eval/train_token_count": 1258291200, "train_eval/train_batch_count": 600, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 18844.378502471023, "train_eval/train_update_time": 10883.880087946192, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.193009602014271, "train_eval/perplexity_len_2048": 66.22179259272077, "train_eval/loss_avg_len_1024": 4.2229134864690785, "train_eval/perplexity_len_1024": 68.23198795379636, "train_eval/loss_avg_len_512": 4.282069746723865, "train_eval/perplexity_len_512": 72.39011426182817}
+{"step": 1363148800, "train_eval/train_token_count": 1363148800, "train_eval/train_batch_count": 650, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 20578.88107217505, "train_eval/train_update_time": 11891.677284248348, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.141280513120828, "train_eval/perplexity_len_2048": 62.883292797405154, "train_eval/loss_avg_len_1024": 4.178984067427664, "train_eval/perplexity_len_1024": 65.29947963586007, "train_eval/loss_avg_len_512": 4.243167548241873, "train_eval/perplexity_len_512": 69.62805312280105}
+{"step": 1468006400, "train_eval/train_token_count": 1468006400, "train_eval/train_batch_count": 700, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 21992.386447268014, "train_eval/train_update_time": 12710.86209141434, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.095283453380916, "train_eval/perplexity_len_2048": 60.05635992330559, "train_eval/loss_avg_len_1024": 4.135403087018712, "train_eval/perplexity_len_1024": 62.5147848907405, "train_eval/loss_avg_len_512": 4.204605343623225, "train_eval/perplexity_len_512": 66.99415277872542}
+{"step": 1572864000, "train_eval/train_token_count": 1572864000, "train_eval/train_batch_count": 750, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 23741.714356134005, "train_eval/train_update_time": 13647.155335442629, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.061455503564557, "train_eval/perplexity_len_2048": 58.05875433420926, "train_eval/loss_avg_len_1024": 4.103027131199469, "train_eval/perplexity_len_1024": 60.523222308524836, "train_eval/loss_avg_len_512": 4.175240817750128, "train_eval/perplexity_len_512": 65.05550429495806}
+{"step": 1677721600, "train_eval/train_token_count": 1677721600, "train_eval/train_batch_count": 800, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 25184.550976835017, "train_eval/train_update_time": 14590.463811040623, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.038684661284387, "train_eval/perplexity_len_2048": 56.75164605324487, "train_eval/loss_avg_len_1024": 4.082819647277592, "train_eval/perplexity_len_1024": 59.31247454945333, "train_eval/loss_avg_len_512": 4.1564677283795755, "train_eval/perplexity_len_512": 63.84560381712993}
+{"step": 1782579200, "train_eval/train_token_count": 1782579200, "train_eval/train_batch_count": 850, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 26014.853294235014, "train_eval/train_update_time": 15036.594645244593, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.0144972322152075, "train_eval/perplexity_len_2048": 55.39543735868876, "train_eval/loss_avg_len_1024": 4.053417672083615, "train_eval/perplexity_len_1024": 57.593958329267856, "train_eval/loss_avg_len_512": 4.128523392552015, "train_eval/perplexity_len_512": 62.086178302728364}
+{"step": 1887436800, "train_eval/train_token_count": 1887436800, "train_eval/train_batch_count": 900, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 26688.83935572201, "train_eval/train_update_time": 15482.672627257823, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 4.009689314002153, "train_eval/perplexity_len_2048": 55.12973986422666, "train_eval/loss_avg_len_1024": 4.05262587939289, "train_eval/perplexity_len_1024": 57.54837390312208, "train_eval/loss_avg_len_512": 4.128599714143057, "train_eval/perplexity_len_512": 62.09091699946827}
+{"step": 1992294400, "train_eval/train_token_count": 1992294400, "train_eval/train_batch_count": 950, "train_eval/train_flop_count": 0, "train_eval/train_total_time": 27477.10213337402, "train_eval/train_update_time": 15928.765013526718, "train_eval/window_seq_count": 51200, "train_eval/window_token_count": 104857600, "train_eval/loss_avg_len_2048": 3.997795303428329, "train_eval/perplexity_len_2048": 54.47791027374324, "train_eval/loss_avg_len_1024": 4.043998843978606, "train_eval/perplexity_len_1024": 57.054037438461634, "train_eval/loss_avg_len_512": 4.119196443598303, "train_eval/perplexity_len_512": 61.509795810391275}

metrics/jsonlines/val.jsonl

@@ -0,0 +1,49 @@
+{"step": 41943040, "val/train_token_count": 41943040, "val/train_batch_count": 20, "val/train_flop_count": 0, "val/train_total_time": 393.3033275610069, "val/train_update_time": 392.7191811740049, "val/loss": 8.011235007830686, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 267.4548993270146, "val/val_tokens_per_second": 153147.3160636276, "val/loss_avg_len_2048": 8.011235007830686, "val/perplexity_len_2048": 3014.637916232049, "val/loss_avg_len_1024": 8.010095824780269, "val/perplexity_len_1024": 3011.205647177279, "val/loss_avg_len_512": 8.010851014741046, "val/perplexity_len_512": 3013.4805383312287}
+{"step": 83886080, "val/train_token_count": 83886080, "val/train_batch_count": 40, "val/train_flop_count": 0, "val/train_total_time": 1014.3213793600444, "val/train_update_time": 745.8892338059959, "val/loss": 7.126910854516412, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 306.0501248690416, "val/val_tokens_per_second": 133834.28618931858, "val/loss_avg_len_2048": 7.126910854516412, "val/perplexity_len_2048": 1245.0249570440558, "val/loss_avg_len_1024": 7.127682446160587, "val/perplexity_len_1024": 1245.9859786081254, "val/loss_avg_len_512": 7.131570649632625, "val/perplexity_len_512": 1250.840056321792}
+{"step": 125829120, "val/train_token_count": 125829120, "val/train_batch_count": 60, "val/train_flop_count": 0, "val/train_total_time": 1633.8735543090152, "val/train_update_time": 1058.99380204099, "val/loss": 6.576425216046255, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 318.1715274940361, "val/val_tokens_per_second": 128735.59215875393, "val/loss_avg_len_2048": 6.576425216046255, "val/perplexity_len_2048": 717.9681552507965, "val/loss_avg_len_1024": 6.578036571076419, "val/perplexity_len_1024": 719.1259894397066, "val/loss_avg_len_512": 6.584267685109842, "val/perplexity_len_512": 723.6209352004278}
+{"step": 167772160, "val/train_token_count": 167772160, "val/train_batch_count": 80, "val/train_flop_count": 0, "val/train_total_time": 2296.034582130029, "val/train_update_time": 1402.5742697739624, "val/loss": 6.157745288895025, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 276.36380792903947, "val/val_tokens_per_second": 148210.43430736448, "val/loss_avg_len_2048": 6.157745288895025, "val/perplexity_len_2048": 472.3618337815753, "val/loss_avg_len_1024": 6.160346911049076, "val/perplexity_len_1024": 473.59234075612176, "val/loss_avg_len_512": 6.168603237254825, "val/perplexity_len_512": 477.5186597915891}
+{"step": 209715200, "val/train_token_count": 209715200, "val/train_batch_count": 100, "val/train_flop_count": 0, "val/train_total_time": 2950.672681943048, "val/train_update_time": 1780.2358716417802, "val/loss": 5.881033806435461, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 242.76836909895064, "val/val_tokens_per_second": 168720.49745205892, "val/loss_avg_len_2048": 5.881033806435461, "val/perplexity_len_2048": 358.17933847684697, "val/loss_avg_len_1024": 5.885018851229502, "val/perplexity_len_1024": 359.60954701480983, "val/loss_avg_len_512": 5.895661722176336, "val/perplexity_len_512": 363.45726407358717}
+{"step": 251658240, "val/train_token_count": 251658240, "val/train_batch_count": 120, "val/train_flop_count": 0, "val/train_total_time": 3595.7228351200465, "val/train_update_time": 2181.321583448793, "val/loss": 5.645073470176547, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 225.88207412901102, "val/val_tokens_per_second": 181333.5571578202, "val/loss_avg_len_2048": 5.645073470176547, "val/perplexity_len_2048": 282.89433975637286, "val/loss_avg_len_1024": 5.6507063047187405, "val/perplexity_len_1024": 284.49233315615174, "val/loss_avg_len_512": 5.663727438149043, "val/perplexity_len_512": 288.22096863377254}
+{"step": 293601280, "val/train_token_count": 293601280, "val/train_batch_count": 140, "val/train_flop_count": 0, "val/train_total_time": 4217.221132909006, "val/train_update_time": 2576.439015968761, "val/loss": 5.478189624139876, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 255.62614679103717, "val/val_tokens_per_second": 160233.9999807725, "val/loss_avg_len_2048": 5.478189624139876, "val/perplexity_len_2048": 239.4128874921015, "val/loss_avg_len_1024": 5.484750836705976, "val/perplexity_len_1024": 240.9888909309404, "val/loss_avg_len_512": 5.498833170221839, "val/perplexity_len_512": 244.40658493943135}
+{"step": 335544320, "val/train_token_count": 335544320, "val/train_batch_count": 160, "val/train_flop_count": 0, "val/train_total_time": 4838.394258918008, "val/train_update_time": 2941.477261595719, "val/loss": 5.316759455512837, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 292.2664110100013, "val/val_tokens_per_second": 140146.10799254093, "val/loss_avg_len_2048": 5.316759455512837, "val/perplexity_len_2048": 203.72263890338473, "val/loss_avg_len_1024": 5.324367502401863, "val/perplexity_len_1024": 205.27848124862788, "val/loss_avg_len_512": 5.339848569961358, "val/perplexity_len_512": 208.48113759190758}
+{"step": 377487360, "val/train_token_count": 377487360, "val/train_batch_count": 180, "val/train_flop_count": 0, "val/train_total_time": 5457.476125103014, "val/train_update_time": 3267.897231984767, "val/loss": 5.191145182319521, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 318.3786125089973, "val/val_tokens_per_second": 128651.85785317938, "val/loss_avg_len_2048": 5.191145182319521, "val/perplexity_len_2048": 179.6741948718429, "val/loss_avg_len_1024": 5.199742717809835, "val/perplexity_len_1024": 181.2256097548097, "val/loss_avg_len_512": 5.216735304176062, "val/perplexity_len_512": 184.3314146769773}
+{"step": 419430400, "val/train_token_count": 419430400, "val/train_batch_count": 200, "val/train_flop_count": 0, "val/train_total_time": 6104.984292554029, "val/train_update_time": 3596.637184852676, "val/loss": 5.0863237302411815, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 290.45753065496683, "val/val_tokens_per_second": 141018.89494012189, "val/loss_avg_len_2048": 5.0863237302411815, "val/perplexity_len_2048": 161.79396911719607, "val/loss_avg_len_1024": 5.095744642082509, "val/perplexity_len_1024": 163.3254183339546, "val/loss_avg_len_512": 5.11419914171081, "val/perplexity_len_512": 166.3674908330965}
+{"step": 461373440, "val/train_token_count": 461373440, "val/train_batch_count": 220, "val/train_flop_count": 0, "val/train_total_time": 6762.651720013004, "val/train_update_time": 3962.670169218676, "val/loss": 4.994484119823133, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 254.08249414898455, "val/val_tokens_per_second": 161207.4855341375, "val/loss_avg_len_2048": 4.994484119823133, "val/perplexity_len_2048": 147.59678348518568, "val/loss_avg_len_1024": 5.004749132925971, "val/perplexity_len_1024": 149.11966925052025, "val/loss_avg_len_512": 5.024751520049106, "val/perplexity_len_512": 152.13244955218386}
+{"step": 503316480, "val/train_token_count": 503316480, "val/train_batch_count": 240, "val/train_flop_count": 0, "val/train_total_time": 7413.931084726995, "val/train_update_time": 4359.249880091811, "val/loss": 4.917526679099561, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 224.98638561402913, "val/val_tokens_per_second": 182055.4603257999, "val/loss_avg_len_2048": 4.917526679099561, "val/perplexity_len_2048": 136.66418045875457, "val/loss_avg_len_1024": 4.928665879917425, "val/perplexity_len_1024": 138.1950205479977, "val/loss_avg_len_512": 4.950037384534534, "val/perplexity_len_512": 141.18024178044544}
+{"step": 545259520, "val/train_token_count": 545259520, "val/train_batch_count": 260, "val/train_flop_count": 0, "val/train_total_time": 8040.308323490026, "val/train_update_time": 4760.0058897937415, "val/loss": 4.846403618092416, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 243.83976028999314, "val/val_tokens_per_second": 167979.1677587248, "val/loss_avg_len_2048": 4.846403618092416, "val/perplexity_len_2048": 127.281811724418, "val/loss_avg_len_1024": 4.85831240278692, "val/perplexity_len_1024": 128.80664484962617, "val/loss_avg_len_512": 4.881023855017313, "val/perplexity_len_512": 131.7655036178709}
+{"step": 587202560, "val/train_token_count": 587202560, "val/train_batch_count": 280, "val/train_flop_count": 0, "val/train_total_time": 8661.466409554007, "val/train_update_time": 5136.805537326669, "val/loss": 4.785894058128424, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 277.76102412998443, "val/val_tokens_per_second": 147464.89406962966, "val/loss_avg_len_2048": 4.785894058128424, "val/perplexity_len_2048": 119.80843091580816, "val/loss_avg_len_1024": 4.7990697121755685, "val/perplexity_len_1024": 121.39743042018526, "val/loss_avg_len_512": 4.823691271648556, "val/perplexity_len_512": 124.4235251872631}
+{"step": 629145600, "val/train_token_count": 629145600, "val/train_batch_count": 300, "val/train_flop_count": 0, "val/train_total_time": 9281.588875080051, "val/train_update_time": 5478.766576015565, "val/loss": 4.726049356866, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 318.06440460996237, "val/val_tokens_per_second": 128778.94981750201, "val/loss_avg_len_2048": 4.726049356866, "val/perplexity_len_2048": 112.84885498484593, "val/loss_avg_len_1024": 4.740272974298615, "val/perplexity_len_1024": 114.4654435400183, "val/loss_avg_len_512": 4.766912040895224, "val/perplexity_len_512": 117.55567379454433}
+{"step": 671088640, "val/train_token_count": 671088640, "val/train_batch_count": 320, "val/train_flop_count": 0, "val/train_total_time": 9914.599941182008, "val/train_update_time": 5792.780345006497, "val/loss": 4.670938681109856, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 304.96344034397043, "val/val_tokens_per_second": 134311.18154294472, "val/loss_avg_len_2048": 4.670938681109856, "val/perplexity_len_2048": 106.79794460932423, "val/loss_avg_len_1024": 4.68608565857932, "val/perplexity_len_1024": 108.42792413784692, "val/loss_avg_len_512": 4.7141645993726335, "val/perplexity_len_512": 111.51561203756256}
+{"step": 713031680, "val/train_token_count": 713031680, "val/train_batch_count": 340, "val/train_flop_count": 0, "val/train_total_time": 10574.152368383016, "val/train_update_time": 6146.868059828412, "val/loss": 4.6206715288446985, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 265.6431397149572, "val/val_tokens_per_second": 154191.82307493905, "val/loss_avg_len_2048": 4.6206715288446985, "val/perplexity_len_2048": 101.56221118922583, "val/loss_avg_len_1024": 4.637006295820186, "val/perplexity_len_1024": 103.23482997037543, "val/loss_avg_len_512": 4.6670295943367295, "val/perplexity_len_512": 106.38127710196552}
+{"step": 754974720, "val/train_token_count": 754974720, "val/train_batch_count": 360, "val/train_flop_count": 0, "val/train_total_time": 11226.902240763011, "val/train_update_time": 6533.353484157473, "val/loss": 4.57530604224482, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 234.03315117699094, "val/val_tokens_per_second": 175017.93995425635, "val/loss_avg_len_2048": 4.57530604224482, "val/perplexity_len_2048": 97.05773836021801, "val/loss_avg_len_1024": 4.59291932442165, "val/perplexity_len_1024": 98.78238747321159, "val/loss_avg_len_512": 4.6248253861254085, "val/perplexity_len_512": 101.98496353821467}
+{"step": 796917760, "val/train_token_count": 796917760, "val/train_batch_count": 380, "val/train_flop_count": 0, "val/train_total_time": 11862.473320519028, "val/train_update_time": 6934.267906721507, "val/loss": 4.536200403668335, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 234.15100623102626, "val/val_tokens_per_second": 174929.84830305027, "val/loss_avg_len_2048": 4.536200403668335, "val/perplexity_len_2048": 93.3354883471502, "val/loss_avg_len_1024": 4.555115518093761, "val/perplexity_len_1024": 95.11774242124372, "val/loss_avg_len_512": 4.5892477317730895, "val/perplexity_len_512": 98.42036379398839}
+{"step": 838860800, "val/train_token_count": 838860800, "val/train_batch_count": 400, "val/train_flop_count": 0, "val/train_total_time": 12483.889439035032, "val/train_update_time": 7321.039978560642, "val/loss": 4.493805073933746, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 265.7580173549941, "val/val_tokens_per_second": 154125.17149120086, "val/loss_avg_len_2048": 4.493805073933746, "val/perplexity_len_2048": 89.46120556473534, "val/loss_avg_len_1024": 4.514267320341524, "val/perplexity_len_1024": 91.3106400591625, "val/loss_avg_len_512": 4.5507813966460535, "val/perplexity_len_512": 94.70638265927766}
+{"step": 880803840, "val/train_token_count": 880803840, "val/train_batch_count": 420, "val/train_flop_count": 0, "val/train_total_time": 13104.464458574017, "val/train_update_time": 7674.994709033519, "val/loss": 4.453331883506127, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 305.22737336700084, "val/val_tokens_per_second": 134195.04138231504, "val/loss_avg_len_2048": 4.453331883506127, "val/perplexity_len_2048": 85.91271882443378, "val/loss_avg_len_1024": 4.475549799342454, "val/perplexity_len_1024": 87.84288309500913, "val/loss_avg_len_512": 4.514732480961177, "val/perplexity_len_512": 91.35312405324778}
+{"step": 922746880, "val/train_token_count": 922746880, "val/train_batch_count": 440, "val/train_flop_count": 0, "val/train_total_time": 13721.99477496103, "val/train_update_time": 7986.941047978529, "val/loss": 4.41391773877088, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 318.02968962700106, "val/val_tokens_per_second": 128793.00686687352, "val/loss_avg_len_2048": 4.41391773877088, "val/perplexity_len_2048": 82.59240596006008, "val/loss_avg_len_1024": 4.437905768002942, "val/perplexity_len_1024": 84.59758905201315, "val/loss_avg_len_512": 4.480083948131558, "val/perplexity_len_512": 88.24208012247752}
+{"step": 964689920, "val/train_token_count": 964689920, "val/train_batch_count": 460, "val/train_flop_count": 0, "val/train_total_time": 14381.773025262053, "val/train_update_time": 8328.335106454615, "val/loss": 4.375631547093648, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 277.73073812999064, "val/val_tokens_per_second": 147480.9748312009, "val/loss_avg_len_2048": 4.375631547093648, "val/perplexity_len_2048": 79.4900253977448, "val/loss_avg_len_1024": 4.401483430209105, "val/perplexity_len_1024": 81.57178500740743, "val/loss_avg_len_512": 4.44655481751617, "val/perplexity_len_512": 85.3324511366961}
+{"step": 1006632960, "val/train_token_count": 1006632960, "val/train_batch_count": 480, "val/train_flop_count": 0, "val/train_total_time": 15035.86255901301, "val/train_update_time": 8704.087580051564, "val/loss": 4.339762180510187, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 244.46324991004076, "val/val_tokens_per_second": 167550.7464417361, "val/loss_avg_len_2048": 4.339762180510187, "val/perplexity_len_2048": 76.6892989594701, "val/loss_avg_len_1024": 4.3674431190324015, "val/perplexity_len_1024": 78.84178470140286, "val/loss_avg_len_512": 4.415505284006894, "val/perplexity_len_512": 82.72362927458667}
+{"step": 1048576000, "val/train_token_count": 1048576000, "val/train_batch_count": 500, "val/train_flop_count": 0, "val/train_total_time": 15681.829017209006, "val/train_update_time": 9104.969846359396, "val/loss": 4.306412981705158, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 225.08503117703367, "val/val_tokens_per_second": 181975.6728637551, "val/loss_avg_len_2048": 4.306412981705158, "val/perplexity_len_2048": 74.17394787427155, "val/loss_avg_len_1024": 4.335562960951775, "val/perplexity_len_1024": 76.3679389581835, "val/loss_avg_len_512": 4.386069002373144, "val/perplexity_len_512": 80.32404392703333}
+{"step": 1090519040, "val/train_token_count": 1090519040, "val/train_batch_count": 520, "val/train_flop_count": 0, "val/train_total_time": 16303.791181104025, "val/train_update_time": 9500.859303092468, "val/loss": 4.2748331887701765, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 254.23934659303632, "val/val_tokens_per_second": 161108.0289061831, "val/loss_avg_len_2048": 4.2748331887701765, "val/perplexity_len_2048": 71.8681498952332, "val/loss_avg_len_1024": 4.306274790479569, "val/perplexity_len_1024": 74.16369839371849, "val/loss_avg_len_512": 4.360181407163851, "val/perplexity_len_512": 78.27133211563368}
+{"step": 1132462080, "val/train_token_count": 1132462080, "val/train_batch_count": 540, "val/train_flop_count": 0, "val/train_total_time": 16924.293944418023, "val/train_update_time": 9866.63232135144, "val/loss": 4.246361440885579, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 290.5493878980051, "val/val_tokens_per_second": 140974.31179025117, "val/loss_avg_len_2048": 4.246361440885579, "val/perplexity_len_2048": 69.85079316400844, "val/loss_avg_len_1024": 4.279776382185658, "val/perplexity_len_1024": 72.22428756407818, "val/loss_avg_len_512": 4.336207040463295, "val/perplexity_len_512": 76.41714182658222}
+{"step": 1174405120, "val/train_token_count": 1174405120, "val/train_batch_count": 560, "val/train_flop_count": 0, "val/train_total_time": 17543.088614015025, "val/train_update_time": 10194.5136101502, "val/loss": 4.216620016333461, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 317.9768462689826, "val/val_tokens_per_second": 128814.41048494192, "val/loss_avg_len_2048": 4.216620016333461, "val/perplexity_len_2048": 67.8039204033866, "val/loss_avg_len_1024": 4.251321952285059, "val/perplexity_len_1024": 70.19814964036787, "val/loss_avg_len_512": 4.309930654432904, "val/perplexity_len_512": 74.43532700140584}
+{"step": 1216348160, "val/train_token_count": 1216348160, "val/train_batch_count": 580, "val/train_flop_count": 0, "val/train_total_time": 18186.98511047603, "val/train_update_time": 10520.074710074114, "val/loss": 4.190980965757324, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 292.99493647099007, "val/val_tokens_per_second": 139797.63777950313, "val/loss_avg_len_2048": 4.190980965757324, "val/perplexity_len_2048": 66.08758883459666, "val/loss_avg_len_1024": 4.227650031735189, "val/perplexity_len_1024": 68.55593845078425, "val/loss_avg_len_512": 4.28926362375198, "val/perplexity_len_512": 72.91275750350573}
+{"step": 1258291200, "val/train_token_count": 1258291200, "val/train_batch_count": 600, "val/train_flop_count": 0, "val/train_total_time": 18844.378502471023, "val/train_update_time": 10883.880087946192, "val/loss": 4.169307191771875, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 256.15296956995735, "val/val_tokens_per_second": 159904.45111280863, "val/loss_avg_len_2048": 4.169307191771875, "val/perplexity_len_2048": 64.67063223727632, "val/loss_avg_len_1024": 4.206767542274017, "val/perplexity_len_1024": 67.13916416068649, "val/loss_avg_len_512": 4.269754838026688, "val/perplexity_len_512": 71.50410338310292}
+{"step": 1300234240, "val/train_token_count": 1300234240, "val/train_batch_count": 620, "val/train_flop_count": 0, "val/train_total_time": 19496.670543703018, "val/train_update_time": 11278.836069991172, "val/loss": 4.146260214753799, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 226.11030463204952, "val/val_tokens_per_second": 181150.52326630766, "val/loss_avg_len_2048": 4.146260214753799, "val/perplexity_len_2048": 63.19721380099956, "val/loss_avg_len_1024": 4.185796775772237, "val/perplexity_len_1024": 65.74586476501416, "val/loss_avg_len_512": 4.251360247742944, "val/perplexity_len_512": 70.20083796212602}
+{"step": 1342177280, "val/train_token_count": 1342177280, "val/train_batch_count": 640, "val/train_flop_count": 0, "val/train_total_time": 20124.23348677403, "val/train_update_time": 11679.664113470295, "val/loss": 4.126677757895738, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 242.41475300304592, "val/val_tokens_per_second": 168966.61400589487, "val/loss_avg_len_2048": 4.126677757895738, "val/perplexity_len_2048": 61.97169557951531, "val/loss_avg_len_1024": 4.167139344774094, "val/perplexity_len_1024": 64.53058805336352, "val/loss_avg_len_512": 4.234325625280757, "val/perplexity_len_512": 69.015120983761}
+{"step": 1384120320, "val/train_token_count": 1384120320, "val/train_batch_count": 660, "val/train_flop_count": 0, "val/train_total_time": 20744.93009287404, "val/train_update_time": 12057.427525710431, "val/loss": 4.10886636920611, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 275.83039805601584, "val/val_tokens_per_second": 148497.04850761886, "val/loss_avg_len_2048": 4.10886636920611, "val/perplexity_len_2048": 60.87766564089016, "val/loss_avg_len_1024": 4.150810060911067, "val/perplexity_len_1024": 63.485406520472054, "val/loss_avg_len_512": 4.219681530099177, "val/perplexity_len_512": 68.01182112214181}
+{"step": 1426063360, "val/train_token_count": 1426063360, "val/train_batch_count": 680, "val/train_flop_count": 0, "val/train_total_time": 21364.400057210005, "val/train_update_time": 12400.716242285387, "val/loss": 4.093701358500426, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 317.5061725229607, "val/val_tokens_per_second": 129005.36602020846, "val/loss_avg_len_2048": 4.093701358500426, "val/perplexity_len_2048": 59.961420185369775, "val/loss_avg_len_1024": 4.136613275307044, "val/perplexity_len_1024": 62.59048534792726, "val/loss_avg_len_512": 4.206974884290714, "val/perplexity_len_512": 67.15308637366292}
+{"step": 1468006400, "val/train_token_count": 1468006400, "val/train_batch_count": 700, "val/train_flop_count": 0, "val/train_total_time": 21992.386447268014, "val/train_update_time": 12710.86209141434, "val/loss": 4.0797728683573435, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 307.7034010089701, "val/val_tokens_per_second": 133115.2007605075, "val/loss_avg_len_2048": 4.0797728683573435, "val/perplexity_len_2048": 59.13203756766548, "val/loss_avg_len_1024": 4.123041846866394, "val/perplexity_len_1024": 61.74678114027358, "val/loss_avg_len_512": 4.1944116867515255, "val/perplexity_len_512": 66.31470627859792}
+{"step": 1509949440, "val/train_token_count": 1509949440, "val/train_batch_count": 720, "val/train_flop_count": 0, "val/train_total_time": 22653.044829952996, "val/train_update_time": 13062.762519414595, "val/loss": 4.067890912848758, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 267.4376528779976, "val/val_tokens_per_second": 153157.19218746488, "val/loss_avg_len_2048": 4.067890912848758, "val/perplexity_len_2048": 58.43359100092499, "val/loss_avg_len_1024": 4.11147258811649, "val/perplexity_len_1024": 61.036533096062215, "val/loss_avg_len_512": 4.183689289411436, "val/perplexity_len_512": 65.60745215431737}
+{"step": 1551892480, "val/train_token_count": 1551892480, "val/train_batch_count": 740, "val/train_flop_count": 0, "val/train_total_time": 23305.888173635, "val/train_update_time": 13447.544946793583, "val/loss": 4.056387900622422, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 235.9053416660172, "val/val_tokens_per_second": 173628.96367980124, "val/loss_avg_len_2048": 4.056387900622422, "val/perplexity_len_2048": 57.765279854047876, "val/loss_avg_len_1024": 4.101134180352232, "val/perplexity_len_1024": 60.4087631904987, "val/loss_avg_len_512": 4.17459845949933, "val/perplexity_len_512": 65.01372877387615}
+{"step": 1593835520, "val/train_token_count": 1593835520, "val/train_batch_count": 760, "val/train_flop_count": 0, "val/train_total_time": 23943.319328956015, "val/train_update_time": 13848.439878219564, "val/loss": 4.046861652540835, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 233.5193387119798, "val/val_tokens_per_second": 175403.03182564085, "val/loss_avg_len_2048": 4.046861652540835, "val/perplexity_len_2048": 57.21760624672959, "val/loss_avg_len_1024": 4.0928021227691325, "val/perplexity_len_1024": 59.90752496972356, "val/loss_avg_len_512": 4.167273688912205, "val/perplexity_len_512": 64.53925794195865}
+{"step": 1635778560, "val/train_token_count": 1635778560, "val/train_batch_count": 780, "val/train_flop_count": 0, "val/train_total_time": 24564.46875228104, "val/train_update_time": 14235.556923599623, "val/loss": 4.038371388106723, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 264.80620891199214, "val/val_tokens_per_second": 154679.15260858924, "val/loss_avg_len_2048": 4.038371388106723, "val/perplexity_len_2048": 56.73387006926299, "val/loss_avg_len_1024": 4.084637223889167, "val/perplexity_len_1024": 59.42037754724601, "val/loss_avg_len_512": 4.159778234884888, "val/perplexity_len_512": 64.05731534674663}
+{"step": 1677721600, "val/train_token_count": 1677721600, "val/train_batch_count": 800, "val/train_flop_count": 0, "val/train_total_time": 25184.550976835017, "val/train_update_time": 14590.463811040623, "val/loss": 4.031109951576823, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 155.41316219599685, "val/val_tokens_per_second": 263555.5407356292, "val/loss_avg_len_2048": 4.031109951576823, "val/perplexity_len_2048": 56.32339280361645, "val/loss_avg_len_1024": 4.077683951509232, "val/perplexity_len_1024": 59.00864458184025, "val/loss_avg_len_512": 4.153272050888557, "val/perplexity_len_512": 63.64189951809101}
+{"step": 1719664640, "val/train_token_count": 1719664640, "val/train_batch_count": 820, "val/train_flop_count": 0, "val/train_total_time": 25519.29751138203, "val/train_update_time": 14768.911977301526, "val/loss": 4.024918882568856, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 113.86328060599044, "val/val_tokens_per_second": 359729.6668601788, "val/loss_avg_len_2048": 4.024918882568856, "val/perplexity_len_2048": 55.97576798697239, "val/loss_avg_len_1024": 4.0717748931922015, "val/perplexity_len_1024": 58.660987235159695, "val/loss_avg_len_512": 4.147944821586087, "val/perplexity_len_512": 63.303765983218696}
+{"step": 1761607680, "val/train_token_count": 1761607680, "val/train_batch_count": 840, "val/train_flop_count": 0, "val/train_total_time": 25811.900586274045, "val/train_update_time": 14947.35764870455, "val/loss": 4.020128366824519, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 113.57747345400276, "val/val_tokens_per_second": 360634.8931206698, "val/loss_avg_len_2048": 4.020128366824519, "val/perplexity_len_2048": 55.708256459815, "val/loss_avg_len_1024": 4.067610871110111, "val/perplexity_len_1024": 58.41722944757159, "val/loss_avg_len_512": 4.144215765030589, "val/perplexity_len_512": 63.06814225983229}
+{"step": 1803550720, "val/train_token_count": 1803550720, "val/train_batch_count": 860, "val/train_flop_count": 0, "val/train_total_time": 26104.19660033105, "val/train_update_time": 15125.80504047859, "val/loss": 4.015582729529799, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 113.59510378103005, "val/val_tokens_per_second": 360578.9214203805, "val/loss_avg_len_2048": 4.015582729529799, "val/perplexity_len_2048": 55.4556016053387, "val/loss_avg_len_1024": 4.063153423574707, "val/perplexity_len_1024": 58.15741719207481, "val/loss_avg_len_512": 4.140098871274945, "val/perplexity_len_512": 62.809031151203996}
+{"step": 1845493760, "val/train_token_count": 1845493760, "val/train_batch_count": 880, "val/train_flop_count": 0, "val/train_total_time": 26396.506954086013, "val/train_update_time": 15304.24179791665, "val/loss": 4.012362551404745, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 113.62905184895499, "val/val_tokens_per_second": 360471.1940608937, "val/loss_avg_len_2048": 4.012362551404745, "val/perplexity_len_2048": 55.277311906496806, "val/loss_avg_len_1024": 4.060099166782666, "val/perplexity_len_1024": 57.980060489936854, "val/loss_avg_len_512": 4.137323217885289, "val/perplexity_len_512": 62.63493677558229}
+{"step": 1887436800, "val/train_token_count": 1887436800, "val/train_batch_count": 900, "val/train_flop_count": 0, "val/train_total_time": 26688.83935572201, "val/train_update_time": 15482.672627257823, "val/loss": 4.0098582680169725, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 113.5941264309804, "val/val_tokens_per_second": 360582.02379757044, "val/loss_avg_len_2048": 4.0098582680169725, "val/perplexity_len_2048": 55.139055042008856, "val/loss_avg_len_1024": 4.057653654730785, "val/perplexity_len_1024": 57.83844278771777, "val/loss_avg_len_512": 4.134992124096118, "val/perplexity_len_512": 62.48909891039072}
+{"step": 1929379840, "val/train_token_count": 1929379840, "val/train_batch_count": 920, "val/train_flop_count": 0, "val/train_total_time": 26981.709376943007, "val/train_update_time": 15661.105290838808, "val/loss": 4.0080952594994805, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 113.74540637602331, "val/val_tokens_per_second": 360102.4542880711, "val/loss_avg_len_2048": 4.0080952594994805, "val/perplexity_len_2048": 55.04193005956597, "val/loss_avg_len_1024": 4.055931827918859, "val/perplexity_len_1024": 57.73894069343542, "val/loss_avg_len_512": 4.133420608581789, "val/perplexity_len_512": 62.3909734450169}
+{"step": 1971322880, "val/train_token_count": 1971322880, "val/train_batch_count": 940, "val/train_flop_count": 0, "val/train_total_time": 27274.170978915005, "val/train_update_time": 15839.539326993749, "val/loss": 4.006876775859995, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 113.56194219802273, "val/val_tokens_per_second": 360684.2152151319, "val/loss_avg_len_2048": 4.006876775859995, "val/perplexity_len_2048": 54.97490321215408, "val/loss_avg_len_1024": 4.054881287327689, "val/perplexity_len_1024": 57.67831544276481, "val/loss_avg_len_512": 4.132452442725096, "val/perplexity_len_512": 62.33059786626467}
+{"step": 2013265920, "val/train_token_count": 2013265920, "val/train_batch_count": 960, "val/train_flop_count": 0, "val/train_total_time": 27566.45336975, "val/train_update_time": 16017.985979812802, "val/loss": 4.006194728950761, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 113.64657631900627, "val/val_tokens_per_second": 360415.608869951, "val/loss_avg_len_2048": 4.006194728950761, "val/perplexity_len_2048": 54.93742053325847, "val/loss_avg_len_1024": 4.05419063684349, "val/perplexity_len_1024": 57.638493639333326, "val/loss_avg_len_512": 4.131800313583575, "val/perplexity_len_512": 62.289963517849074}
+{"step": 2055208960, "val/train_token_count": 2055208960, "val/train_batch_count": 980, "val/train_flop_count": 0, "val/train_total_time": 27858.825974697014, "val/train_update_time": 16196.423335834814, "val/loss": 4.005892366719665, "val/val_token_count": 40960000, "val/val_seq_count": 20000, "val/val_time": 113.76460951002082, "val/val_tokens_per_second": 360041.6700449544, "val/loss_avg_len_2048": 4.005892366719665, "val/perplexity_len_2048": 54.92081204323195, "val/loss_avg_len_1024": 4.053872466314816, "val/perplexity_len_1024": 57.62015768647472, "val/loss_avg_len_512": 4.13152233827468, "val/perplexity_len_512": 62.27265085235736}

metrics/jsonlines/val_data_info.jsonl

@@ -0,0 +1 @@
+{"step": 0, "val_data_info/vocab_size": 50277, "val_data_info/global_tokens_per_batch": 2048, "val_data_info/local_tokens_per_batch": 2048, "val_data_info/batch_len": 2048, "val_data_info/seq_len": 2048, "val_data_info/total_tokens": 2147483648, "val_data_info/global_batch_size": 1, "val_data_info/local_batch_size": 1}

metrics/npz/train_eval/step-000000104857600.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:844d2f397f2e90042ce9a56ae3b082d27ba5526d9b59ee11aeb9391e9a9e557c
+size 20540

metrics/npz/train_eval/step-000000209715200.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d5c1df2e5ee8970ecffecaaf04b8b6b5726d2d851aef320b7759920525395c5
+size 20540