best: model_2907_loss_8.634e-02.chkpt
data:
  args:
    patch_length: 1024
    patch_size: 16
    patch_stream: true
    prompt_dropout: 0.3
    tokenizer_path: assets/lilylet-tokenizer.json
  args_variant:
    1:
      prompt_dropout: 0
  batch_size: 8
  root: 20260617/notagenx1m-lyl-20260617.pt
  splits: '*0..997/1000:998,999/1000'
  type: LilyletPatchy
id: lilylet/20260617-lilylet-notagenx-1m0617-llama-l4+10-lr0.2
model:
  args:
    base_type: llama
    hidden_size: 1024
    intermediate_size: 3072
    n_head: 16
    num_key_value_heads: 16
    patch_length: 1024
    patch_num_layers: 10
    patch_size: 16
    token_num_layers: 4
    token_vocab_size: 256
  type: LilyletNotaGen
optim:
  args:
    betas:
    - 0.9
    - 0.98
    eps: 1.0e-09
  scheduler:
    args:
      d_model: 1024
      lr_mul: 0.2
      n_warmup_steps: 1000
    type: InvSqrt
  type: Adam
trainer:
  device: cuda
  dtype: bf16
  env: null
  epoch_duration: 0
  epoch_size: 800
  epochs: 10000
  latest: true
  moniter:
    best_value: 0.08634062658767311
    field: loss
    mode: min
  report_step_unit: examples
  save_mode: best
  steps: 291200