{ "a_low_rank_dim": 64, "architectures": [ "RWKV7Model" ], "attn": null, "attn_mode": "chunk", "auto_map": { "AutoConfig": "configuration_rwkv7.RWKV7Config", "AutoModel": "modeling_rwkv7.RWKV7Model", "AutoModelForCausalLM": "modeling_rwkv7.RWKV7ForCausalLM" }, "bos_token_id": 2, "decay_low_rank_dim": 64, "dtype": "float32", "eos_token_id": 3, "fuse_cross_entropy": true, "fuse_linear_cross_entropy": false, "fuse_linear_cross_entropyl": false, "fuse_norm": false, "gate_low_rank_dim": 128, "head_dim": 16, "hidden_act": "sqrelu", "hidden_ratio": 4, "hidden_size": 16, "initializer_range": 0.02, "intermediate_size": 32, "max_position_embeddings": 2048, "model_type": "rwkv7", "norm_bias": true, "norm_eps": 1e-05, "norm_first": true, "num_heads": 16, "num_hidden_layers": 2, "pad_token_id": 0, "tie_word_embeddings": false, "transformers_version": "5.22.1", "use_cache": false, "use_l2warp": false, "v_low_rank_dim": 32, "value_dim": [ 16, 16 ], "vocab_size": 512 }