| { | |
| "architectures" : [ | |
| "Step1ForCausalLM" | |
| ], | |
| "auto_map" : { | |
| "AutoConfig" : "configuration_step1.Step1Config", | |
| "AutoModelForCausalLM" : "modeling_step1.Step1ForCausalLM" | |
| }, | |
| "bos_token_id" : 1, | |
| "eos_token_id" : 3, | |
| "hidden_size" : 3072, | |
| "intermediate_size" : 8192, | |
| "max_seq_len" : 32768, | |
| "model_type" : "step1", | |
| "num_attention_groups" : 4, | |
| "num_attention_heads" : 48, | |
| "num_hidden_layers" : 32, | |
| "pad_token_id" : 0, | |
| "quantization" : { | |
| "bits" : 8, | |
| "group_size" : 64, | |
| "mode" : "affine" | |
| }, | |
| "rms_norm_eps" : 1.0000000000000001e-05, | |
| "tie_word_embeddings" : false, | |
| "torch_dtype" : "bfloat16", | |
| "use_cache" : true, | |
| "vocab_size" : 74752 | |
| } |