| { |
| "architectures": [ |
| "TaoNetMiniT2ForCausalLM" |
| ], |
| "auto_map": { |
| "AutoConfig": "configuration_taonet_mini_t2.TaoNetMiniT2Config", |
| "AutoModelForCausalLM": "modeling_taonet_mini_t2.TaoNetMiniT2ForCausalLM", |
| "AutoTokenizer": [ |
| "tokenization_taonet_mini_t2.TaoNetMiniT2Tokenizer", |
| null |
| ] |
| }, |
| "bos_token_id": 1, |
| "checkpoint_name": "pretrain_final_model.pt", |
| "default_ssm_finite_tail_correction": true, |
| "default_ssm_kernel_mode": "recurrent", |
| "eos_token_id": 2, |
| "model_type": "taonet_mini_t2", |
| "pad_token_id": 3, |
| "taotrain_model_config": { |
| "architecture_type": "taonet_ssm", |
| "vocab_size": 8192, |
| "hidden_dim": 1024, |
| "num_layers": 18, |
| "num_heads": 8, |
| "max_seq_length": 512, |
| "d_latent_kv": 768, |
| "d_rope": 128, |
| "hidden_dim_ff": 3072, |
| "dropout": 0.0, |
| "gqa_groups": 1, |
| "use_factorized_embedding": false, |
| "d_embed_rank": 96, |
| "init_std": 0.02, |
| "ssm_core": "dplr", |
| "ssm_hidden_dim": 32, |
| "ssm_mixer_dim": 256, |
| "ssm_num_lanes": 2, |
| "ssm_lane_combine": "channel", |
| "ssm_lane_mode": "split", |
| "ssm_split_mix": "none", |
| "ssm_rank": 1, |
| "ssm_max_low_rank_scale": 0.1, |
| "ssm_finite_tail_correction": true, |
| "ssm_discretization": "bilinear", |
| "ssm_kernel_mode": "recurrent", |
| "ssm_kernel_threshold": 64, |
| "ssm_dt_min": 0.001, |
| "ssm_dt_max": 0.1, |
| "ssm_dt_init": 0.01, |
| "ssm_use_d": true, |
| "ssm_activation": "gelu", |
| "ssm_gate": true, |
| "ssm_input_gate": true, |
| "ssm_gate_type": "channel", |
| "ssm_use_padding_mask": false, |
| "ssm_layer_scale_init": 0.1, |
| "ssm_branch_rms_norm": true, |
| "ssm_branch_rms_eps": 0.000001, |
| "ssm_branch_clip_value": null, |
| "block_residual_rms_norm": false, |
| "block_residual_rms_target": 1.0, |
| "block_residual_rms_cap": null, |
| "block_residual_rms_eps": 0.000001, |
| "ssm_local_shift": true, |
| "ssm_local_shift_init": 0.1, |
| "ssm_local_shift_per_channel": true |
| }, |
| "tokenizer_class": "TaoNetMiniT2Tokenizer", |
| "tokenizer_file": "tokenizer.model", |
| "torch_dtype": "bfloat16", |
| "transformers_version": "4.30.0", |
| "unk_token_id": 0, |
| "use_cache": false, |
| "vocab_size": 8192 |
| } |
|
|
