{ "architecture": "HNet", "config": { "arch_layout": [ "m4", [ "T12" ], "m4" ], "d_model": [ 512, 768 ], "d_intermediate": [ 0, 2048 ], "vocab_size": 256, "ssm_cfg": { "chunk_size": 256, "d_conv": 4, "d_state": 128, "expand": 2 }, "attn_cfg": { "num_heads": [ 8, 12 ], "rotary_emb_dim": [ 32, 32 ], "window_size": [ 1023, -1 ] }, "tie_embeddings": false }, "iter": 72799, "n_params": 99765120, "tied_weights": {}, "training_args": { "config": "configs/hnet_small.json", "block_size": 8192, "micro_bs": 2, "grad_accum": 16, "max_iters": 72800, "lr": 0.0006, "lr_mult": "2.0,1.0", "min_lr_frac": 0.1, "warmup_iters": 2000, "weight_decay": 0.1, "grad_clip": 1.0, "target_n": 4.6, "ratio_weight": 0.03, "eval_every": 500, "eval_windows": 64, "ckpt_every": 1000, "snap_every": 8000, "log_every": 20, "seed": 1337 } }