| { | |
| "model_type": "ana-lm", | |
| "architectures": [ | |
| "ANALMForCausalLM" | |
| ], | |
| "auto_map": { | |
| "AutoConfig": "model.ANALMConfig", | |
| "AutoModelForCausalLM": "model.ANALMForCausalLM" | |
| }, | |
| "vocab_size": 32000, | |
| "layers": 12, | |
| "num_hidden_layers": 12, | |
| "d": 448, | |
| "hidden_size": 448, | |
| "h": 8, | |
| "num_attention_heads": 8, | |
| "max_l": 512, | |
| "max_position_embeddings": 512, | |
| "key_mask": false, | |
| "use_gates": false, | |
| "gate_init": 1.0, | |
| "gate_channels": false, | |
| "ffn_mult": 2.0, | |
| "attn_impl": "sdpa", | |
| "qk_norm": true, | |
| "attn_softcap": null, | |
| "z_loss_coef": 0.0, | |
| "diff_attn": "none", | |
| "architecture": "full", | |
| "use_output_scaling": true, | |
| "bos_token_id": 1, | |
| "eos_token_id": 2, | |
| "pad_token_id": 2, | |
| "torch_dtype": "float16", | |
| "decode_defaults": { | |
| "max_new_tokens": 64, | |
| "temperature": 0.65, | |
| "top_k": 24, | |
| "top_p": 0.9, | |
| "repetition_penalty": 1.1, | |
| "frequency_penalty": 0.03, | |
| "presence_penalty": 0.0, | |
| "no_repeat_ngram": 3, | |
| "history_scope": "generated", | |
| "history_window": 96, | |
| "ban_special_tokens": true, | |
| "min_new_before_eos": 16, | |
| "stop_eos": true, | |
| "strategy": "sample", | |
| "beam_size": 4, | |
| "beam_top_k": 8, | |
| "beam_score_alpha": 1.0 | |
| } | |
| } | |