| { | |
| "architecture": "TinyChatLM", | |
| "model_type": "custom-decoder-only-transformer", | |
| "vocab_size": 8192, | |
| "d_model": 192, | |
| "n_layers": 8, | |
| "n_heads": 6, | |
| "n_kv_heads": 2, | |
| "d_ff": 560, | |
| "max_seq_len": 512, | |
| "dropout": 0.0, | |
| "training": { | |
| "stage": "sft", | |
| "step": 3000, | |
| "val_loss": 3.066343913078308 | |
| } | |
| } | |