| { |
| "architectures": [ |
| "LlamaForCausalLMWithGNN" |
| ], |
| "attention_bias": false, |
| "attention_dropout": 0.0, |
| "bos_token_id": 128000, |
| "eos_token_id": [ |
| 128001, |
| 128008, |
| 128009 |
| ], |
| "gnn_config": { |
| "GIN_after_attention": true, |
| "GIN_after_attention_pre_GIN_norm": true, |
| "GIN_after_attention_skip": true, |
| "GIN_edge_weight_scaling": true, |
| "GIN_hidden_dim_multiplier": 1, |
| "GIN_use_MLP": true, |
| "GIN_use_norm": false, |
| "LlamaAttentionHierarchicalPerceiverAR_use_rope": true, |
| "LlamaAttentionHierarchicalVariant_2_PerceiverAR_use_skip": true, |
| "MLP_type": "standard_MLP", |
| "N_GNN_from_attention_layers": 3, |
| "activation": "prelu", |
| "add_rope": false, |
| "adj_construction_method": "sum", |
| "adj_transform_hidden_dim": 128, |
| "attention_GIN_MLP_GIN_MLP_mode": "shared", |
| "attention_GIN_MLP_GIN_MLP_pre_aggregate": true, |
| "attention_GIN_MLP_GIN_binary_scale": 1.0, |
| "attention_GIN_MLP_GIN_fuse_mode": "epsilon", |
| "attention_GIN_MLP_GIN_learnable_threshold": false, |
| "attention_GIN_MLP_GIN_mode": "default", |
| "attention_GIN_MLP_GIN_sharp_softplus_beta": 10.0, |
| "attention_GIN_MLP_GIN_softmax_temperature": 1.0, |
| "attention_GIN_MLP_GIN_threshold_mode": "none", |
| "attention_GIN_MLP_GIN_threshold_value": 0.2, |
| "attention_GIN_MLP_GIN_top_k_fraction_of_sequence_length": 0.1, |
| "attention_GIN_MLP_GIN_use_ReLU_instead_of_softmax": true, |
| "attention_GIN_MLP_GIN_use_softmax": false, |
| "attention_GIN_MLP_attention_mix_mode": "A", |
| "attention_GIN_MLP_multiplier": 2, |
| "attention_GIN_MLP_o_proj_at_end": false, |
| "attention_GIN_MLP_scoring_hidden_dim": 512, |
| "attention_GIN_MLP_second_order_factor": 0.1, |
| "attention_GIN_MLP_separate_attention": false, |
| "attention_GIN_MLP_use_scoring_fnct": true, |
| "attention_GIN_MLP_use_second_order": false, |
| "attention_epsilon_strategy": "default", |
| "attention_epsilon_uniform_value": 0.5, |
| "combined_norm": false, |
| "continuous_transform_alpha": 10.0, |
| "distance_scaling_method": "power", |
| "distance_weight_strength": 1.0, |
| "dropout": 0.1, |
| "enforce_causality": true, |
| "epsilon_threshold": 0.6, |
| "gnn_logic": "before_MLP", |
| "gnn_mode": "single", |
| "gnn_residual": false, |
| "gnn_type": "causal_gin", |
| "group_tokens_for_coarse_graining": false, |
| "hidden_dim": 155, |
| "hierarchical_enc_dec_type": "PerceiverAR", |
| "initial_sharpening_value": 1.0, |
| "lambda_GNN": 0.5, |
| "lambda_GNN_initial": 0.0, |
| "learnable_aggregate_activation": "softmax", |
| "max_position_embeddings": 2048, |
| "mix_weights_initial": 0.5, |
| "model_type": "", |
| "norm_to_hidden_states": false, |
| "num_latent_layers": 4, |
| "num_latents": 32, |
| "num_latents_list": [ |
| 64, |
| 32, |
| 8 |
| ], |
| "num_layers": 1, |
| "per_head_ff": false, |
| "plot_for_debugging": false, |
| "remove_self_connections": false, |
| "residual_epsilon_strategy": "default", |
| "residual_epsilon_uniform_value": 0.1, |
| "rms_norm_eps": 1e-05, |
| "sharpening_value_init": "value", |
| "soft_masking_initial_threshold": 0.01, |
| "soft_masking_k": 10.0, |
| "threshold": 0.1, |
| "threshold_any_tau": 0.1, |
| "tokenizer": null, |
| "top_k": 8, |
| "use_GNN_from_attention": "none", |
| "use_GNN_from_attention_add_RoPE_at_every_layer": false, |
| "use_differential_attention": false, |
| "use_differential_attention_group_norm": false, |
| "use_distance_scaling": false, |
| "use_fixed_number_of_tokens_per_latent": false, |
| "use_graph_property_modulation": false, |
| "use_graph_property_modulation_with_norm": false, |
| "use_graph_property_modulation_with_norm_use_causal_clustering": true, |
| "use_hierarchical_attention": false, |
| "use_layer_norm": true, |
| "use_layer_norm_in_GIN_MLP": false, |
| "use_no_norm_in_GIN_MLP": false, |
| "use_original_hidden_states": false, |
| "use_original_hidden_states_add_attention": false, |
| "use_projection": true, |
| "use_sharpening": false, |
| "use_soft_masking": false, |
| "zero_below_epsilon_threshold": true |
| }, |
| "head_dim": 128, |
| "hidden_act": "silu", |
| "hidden_size": 3072, |
| "initializer_range": 0.02, |
| "intermediate_size": 8192, |
| "max_position_embeddings": 131072, |
| "mlp_bias": false, |
| "model_type": "llama", |
| "num_attention_heads": 24, |
| "num_hidden_layers": 28, |
| "num_key_value_heads": 8, |
| "pretraining_tp": 1, |
| "rms_norm_eps": 1e-05, |
| "rope_scaling": { |
| "factor": 32.0, |
| "high_freq_factor": 4.0, |
| "low_freq_factor": 1.0, |
| "original_max_position_embeddings": 8192, |
| "rope_type": "llama3" |
| }, |
| "rope_theta": 500000.0, |
| "tie_word_embeddings": true, |
| "torch_dtype": "float32", |
| "transformers_version": "4.46.1", |
| "use_cache": false, |
| "vocab_size": 128256 |
| } |
|
|
