Download config.json from ShadeEngine/bytegpt-hnet-100M: direct link, hf CLI and curl.
- Browser
- Download file 1.12 kB
-
https://huggingface.co/ShadeEngine/bytegpt-hnet-100M/resolve/main/config.json
- Command line
-
hf download hf://ShadeEngine/bytegpt-hnet-100M/config.json
-
curl -L -o config.json https://huggingface.co/ShadeEngine/bytegpt-hnet-100M/resolve/main/config.json
1.12 kB
| { | |
| "architecture": "HNet", | |
| "config": { | |
| "arch_layout": [ | |
| "m4", | |
| [ | |
| "T12" | |
| ], | |
| "m4" | |
| ], | |
| "d_model": [ | |
| 512, | |
| 768 | |
| ], | |
| "d_intermediate": [ | |
| 0, | |
| 2048 | |
| ], | |
| "vocab_size": 256, | |
| "ssm_cfg": { | |
| "chunk_size": 256, | |
| "d_conv": 4, | |
| "d_state": 128, | |
| "expand": 2 | |
| }, | |
| "attn_cfg": { | |
| "num_heads": [ | |
| 8, | |
| 12 | |
| ], | |
| "rotary_emb_dim": [ | |
| 32, | |
| 32 | |
| ], | |
| "window_size": [ | |
| 1023, | |
| -1 | |
| ] | |
| }, | |
| "tie_embeddings": false | |
| }, | |
| "iter": 72799, | |
| "n_params": 99765120, | |
| "tied_weights": {}, | |
| "training_args": { | |
| "config": "configs/hnet_small.json", | |
| "block_size": 8192, | |
| "micro_bs": 2, | |
| "grad_accum": 16, | |
| "max_iters": 72800, | |
| "lr": 0.0006, | |
| "lr_mult": "2.0,1.0", | |
| "min_lr_frac": 0.1, | |
| "warmup_iters": 2000, | |
| "weight_decay": 0.1, | |
| "grad_clip": 1.0, | |
| "target_n": 4.6, | |
| "ratio_weight": 0.03, | |
| "eval_every": 500, | |
| "eval_windows": 64, | |
| "ckpt_every": 1000, | |
| "snap_every": 8000, | |
| "log_every": 20, | |
| "seed": 1337 | |
| } | |
| } |