| { |
| "name": "Lumina-Lite-2.2B Optimized Autoregressive Transformer", |
| "layers": [ |
| { |
| "type": "Embedding", |
| "params": { |
| "num_embeddings": 50272, |
| "embedding_dim": 2048 |
| } |
| }, |
| { |
| "type": "TransformerBlock", |
| "params": { |
| "layers": 28, |
| "hidden_size": 2048, |
| "num_attention_heads": 16, |
| "num_key_value_heads": 4, |
| "head_dim": 128, |
| "mlp_intermediate_size": 5504, |
| "normalization": "RMSNorm", |
| "activation": "SwiGLU", |
| "positional_encoding": "RoPE (Rotary Positional Embeddings)", |
| "attention_type": "Grouped-Query Attention (GQA)" |
| } |
| }, |
| { |
| "type": "Linear", |
| "params": { |
| "in_features": 2048, |
| "out_features": 5504, |
| "description": "SwiGLU Gate and Up Projection" |
| } |
| }, |
| { |
| "type": "Linear", |
| "params": { |
| "in_features": 5504, |
| "out_features": 2048, |
| "description": "MLP Down Projection" |
| } |
| }, |
| { |
| "type": "RMSNorm", |
| "params": { |
| "normalized_shape": 2048, |
| "eps": 0.00001 |
| } |
| }, |
| { |
| "type": "Linear", |
| "params": { |
| "in_features": 2048, |
| "out_features": 50272, |
| "bias": false, |
| "description": "Language Modeling Head" |
| } |
| } |
| ], |
| "explanation": "Lumina-Lite-2.2B utilizes a 'deep-and-narrow' topology to optimize for high-throughput within a 6GB VRAM limit. The implementation of Grouped-Query Attention (GQA) reduces the KV cache footprint by a factor of 4 compared to multi-head attention, enabling longer context windows and larger batch sizes on consumer hardware. SwiGLU activation and RMSNorm provide superior gradient flow and training stability over standard GeLU/LayerNorm, while RoPE ensures the model maintains spatial awareness of tokens without absolute position bias. The architecture is specifically designed to facilitate 4-bit NF4 quantization, making it highly efficient for deployment via bitsandbytes or AutoGPTQ." |
| } |
