| { | |
| "architectures": [ | |
| "SupraMiniWithGate" | |
| ], | |
| "model_type": "supra_mini_hadamard_gs", | |
| "vocab_size": 4096, | |
| "d_model": 128, | |
| "num_layers": 6, | |
| "num_heads": 4, | |
| "d_head": 16, | |
| "max_len": 256, | |
| "inner_dim": 64, | |
| "intermediate_dim": 64, | |
| "hadamard_dim": 128, | |
| "total_parameters": 812800, | |
| "tied_embeddings": true, | |
| "dynamic_gram_schmidt": { | |
| "enabled": true, | |
| "gate_projection": "Dense(128, bias=True) -> Sigmoid", | |
| "formula": "h_orth = h_final - (sigmoid(W_gate @ h_final + b) * proj_{x_in}(h_final))" | |
| }, | |
| "sublayer_diffusion": { | |
| "projection": "Pad zeros (64 -> 128)", | |
| "permutation_seed": 2026, | |
| "isometry": "Sylvester Hadamard 128 (normalized 1/sqrt(128))" | |
| }, | |
| "base_tokenizer": "SupraLabs/Supra-Mini-v6-1M", | |
| "training_history": { | |
| "stage_1_tinystories": { | |
| "tokens": 10000000, | |
| "final_loss": 2.7381, | |
| "optimizer": "AdamW 1e-3 -> 5e-4" | |
| }, | |
| "stage_2_general_mix": { | |
| "tokens": 20000000, | |
| "final_loss": 4.7582, | |
| "optimizer": "AdamW 1e-3 -> 1e-4" | |
| }, | |
| "stage_3_annealing": { | |
| "tokens": 50000000, | |
| "final_loss": 4.25, | |
| "optimizer": "AdamW 5e-4 -> 5e-5" | |
| }, | |
| "stage_4_self_distill": { | |
| "tokens": 20000000, | |
| "loss_type": "0.65 One-Hot + 0.35 Top-5 Soft-CE", | |
| "lr": 0.0001 | |
| } | |
| } | |
| } |