| # LLaMA 77M params | |
| # huggingface/transformers/.. models/llama/configuration_llama.py | |
| model_type = "llama" | |
| tie_word_embeddings = true | |
| hidden_size = 768 # embedding dimension | |
| intermediate_size = 2048 # FFN hidden size (typically 4x hidden_size) | |
| max_position_embeddings = 1024 | |
| num_attention_heads = 8 # attention heads (must divide hidden_size evenly) | |
| num_hidden_layers = 8 # transformer layers | |
| num_key_value_heads = 4 | |
| vocab_size = 32768 | |