# LLaMA 77M params
# huggingface/transformers/.. models/llama/configuration_llama.py

model_type = "llama"
tie_word_embeddings = true
hidden_size = 768          # embedding dimension
intermediate_size = 2048   # FFN hidden size (typically 4x hidden_size)
max_position_embeddings = 1024
num_attention_heads = 8    # attention heads (must divide hidden_size evenly)
num_hidden_layers = 8      # transformer layers
num_key_value_heads = 4
vocab_size = 32768