| config = Qwen3Config( | |
| vocab_size=256, | |
| hidden_size=64, | |
| head_dim=16, | |
| num_attention_heads=2, | |
| num_key_value_heads=2, | |
| num_hidden_layers=3, | |
| intermediate_size=160, | |
| max_position_embeddings=1024, | |
| tie_word_embeddings=True, | |
| attention_dropout=0.0, | |
| ) | |
| B, T = 1024, 128 | |
| base_lr = 6e-3 | |
| epochs=1 | |
| opt = torch.optim.AdamW(raw_model.parameters(), lr=base_lr, | |
| betas=(0.9, 0.95), weight_decay=0.1, | |
| fused=(device == "cuda")) | |
| warmup = 1000 | |
| trained on 99.5% of bigfile |