| best: model_2907_loss_8.634e-02.chkpt | |
| data: | |
| args: | |
| patch_length: 1024 | |
| patch_size: 16 | |
| patch_stream: true | |
| prompt_dropout: 0.3 | |
| tokenizer_path: assets/lilylet-tokenizer.json | |
| args_variant: | |
| 1: | |
| prompt_dropout: 0 | |
| batch_size: 8 | |
| root: 20260617/notagenx1m-lyl-20260617.pt | |
| splits: '*0..997/1000:998,999/1000' | |
| type: LilyletPatchy | |
| id: lilylet/20260617-lilylet-notagenx-1m0617-llama-l4+10-lr0.2 | |
| model: | |
| args: | |
| base_type: llama | |
| hidden_size: 1024 | |
| intermediate_size: 3072 | |
| n_head: 16 | |
| num_key_value_heads: 16 | |
| patch_length: 1024 | |
| patch_num_layers: 10 | |
| patch_size: 16 | |
| token_num_layers: 4 | |
| token_vocab_size: 256 | |
| type: LilyletNotaGen | |
| optim: | |
| args: | |
| betas: | |
| - 0.9 | |
| - 0.98 | |
| eps: 1.0e-09 | |
| scheduler: | |
| args: | |
| d_model: 1024 | |
| lr_mul: 0.2 | |
| n_warmup_steps: 1000 | |
| type: InvSqrt | |
| type: Adam | |
| trainer: | |
| device: cuda | |
| dtype: bf16 | |
| env: null | |
| epoch_duration: 0 | |
| epoch_size: 800 | |
| epochs: 10000 | |
| latest: true | |
| moniter: | |
| best_value: 0.08634062658767311 | |
| field: loss | |
| mode: min | |
| report_step_unit: examples | |
| save_mode: best | |
| steps: 291200 | |