config = Qwen3Config(
    vocab_size=256,
    hidden_size=64,
    head_dim=16,
    num_attention_heads=2,
    num_key_value_heads=2,
    num_hidden_layers=3,
    intermediate_size=160,
    max_position_embeddings=1024,
    tie_word_embeddings=True,
    attention_dropout=0.0,
)

B, T = 1024, 128
base_lr = 6e-3
epochs=1
opt = torch.optim.AdamW(raw_model.parameters(), lr=base_lr,
                        betas=(0.9, 0.95), weight_decay=0.1,
                        fused=(device == "cuda"))
warmup = 1000

trained on 99.5% of bigfile