| { | |
| "model_type": "nano-proofread", | |
| "architecture": "decoder-only transformer (pre-norm)", | |
| "vocab_size": 256, | |
| "tokenizer": "byte (raw UTF-8 bytes, no vocab file)", | |
| "dim": 128, | |
| "n_layers": 4, | |
| "n_heads": 4, | |
| "n_kv_heads": 2, | |
| "head_dim": 32, | |
| "ffn": "swiglu", | |
| "ffn_mult": 4, | |
| "norm": "rmsnorm", | |
| "norm_eps": 1e-05, | |
| "positional": "rope", | |
| "rope_theta": 10000.0, | |
| "max_seq_len": 64, | |
| "tie_word_embeddings": true, | |
| "params": 1016960, | |
| "training": { | |
| "task": "short phrase with one common writing error -> corrected phrase", | |
| "scope": "fixed set of context-dependent confusions (their/there/they're, your/you're, its/it's, then/than, to/too, could have/could of) plus doubled words; NOT general grammar correction", | |
| "data": "100% code-generated: build a grammatically-correct phrase from templates with rich slot fillers, then inject one error (swap a confusion word for a wrong family member, or double a word); ~15% identity; label correct by construction", | |
| "objective": "SFT, prompt-masked cross-entropy (only the corrected phrase + newline EOS supervised)", | |
| "steps": 16000, | |
| "batch_size": 64, | |
| "seq_len": 64, | |
| "optimizer": "adamw", | |
| "lr": 0.003, | |
| "schedule": "cosine", | |
| "warmup_steps": 200, | |
| "seed": 0 | |
| } | |
| } | |