config.json

{
    "model_type": "trm",
    "architectures": [
        "TinyRecursiveModel"
    ],
    "torch_dtype": "float32",
    "vocab_size": 50257,
    "dim": 256,
    "n_heads": 8,
    "n_layers": 2,
    "mlp_ratio": 4,
    "max_seq_len": 512,
    "n_latent_recursions": 4,
    "n_improvement_cycles": 2,
    "batch_size": 4,
    "epochs": 1,
    "lr": 0.0001,
    "warmup_steps": 1000,
    "n_supervision_steps": 3,
    "max_train_samples": null,
    "max_val_samples": null
}