Upload 13 files

config.used.json

@@ -0,0 +1,50 @@
+{
+  "architectures": [
+    "MiniMaxForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "layer_types": [
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "full_attention"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_minimax.MiniMaxConfig",
+    "AutoModelForCausalLM": "modeling_minimax.MiniMaxForCausalLM"
+  },
+  "bos_token_id": null,
+  "eos_token_id": 200020,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 256,
+  "initializer_range": 0.02,
+  "intermediate_size": 512,
+  "layernorm_full_attention_alpha": 3.5565588200778455,
+  "layernorm_full_attention_beta": 1.0,
+  "layernorm_linear_attention_alpha": 3.5565588200778455,
+  "layernorm_linear_attention_beta": 1.0,
+  "layernorm_mlp_alpha": 3.5565588200778455,
+  "layernorm_mlp_beta": 1.0,
+  "max_position_embeddings": 1024,
+  "model_type": "minimax",
+  "num_attention_heads": 4,
+  "num_experts_per_tok": 1,
+  "num_hidden_layers": 4,
+  "num_key_value_heads": 2,
+  "num_local_experts": 2,
+  "output_router_logits": false,
+  "postnorm": true,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000,
+  "rotary_dim": 16,
+  "router_aux_loss_coef": 0.001,
+  "router_jitter_noise": 0.0,
+  "shared_intermediate_size": 0,
+  "shared_moe_mode": "sigmoid",
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.45.2",
+  "use_cache": true,
+  "vocab_size": 200064
+}

create.py

@@ -2,6 +2,7 @@ import sys
 import os
 from safetensors.torch import save_file
 import json
+import torch
 
 # Add the directory containing your modeling.py and configuration.py to the Python path
 model_dir = "/Users/Goekdeniz.Guelmez@computacenter.com/Library/CloudStorage/OneDrive-COMPUTACENTER/Desktop/MiniMax01Text-Dev"
@@ -19,9 +20,21 @@ with open(config_path, 'r') as f:
 # Create the configuration object
 config = MiniMaxConfig(**config_dict)
 
+# Print attention layout info
+if getattr(config, "linear_attention", False):
+    print("Using linear attention layout from config.")
+else:
+    print("Using full attention layout from config.")
+
+# Set random seed for reproducibility
+torch.manual_seed(42)
+
 # Create the model
 small_model = MiniMaxForCausalLM(config)
 
+# Set model to evaluation mode
+small_model.eval()
+
 # Print parameter count to verify
 param_count = sum(p.numel() for p in small_model.parameters())
 print(f"Model has {param_count:,} parameters")
@@ -29,7 +42,14 @@ print(f"Model has {param_count:,} parameters")
 # Convert model to state dict
 model_state_dict = small_model.state_dict()
 
+# Save the config used for reproducibility
+used_config_path = os.path.join(model_dir, "config.used.json")
+with open(used_config_path, 'w') as f:
+    json.dump(config_dict, f, indent=2)
+
 # Save as safetensors
 save_file(model_state_dict, os.path.join(model_dir, "model.safetensors"))
 
-print("Model saved in safetensors format")
+print("Model saved in safetensors format")
+
+print(small_model)

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9c7797546b0e092d6b3236d1f9826af42bfe293592590e2032aafe77ba8592a4
-size 423910680
+oid sha256:a70cdf1018eab074427a9a85060958141f22c3346cd377869c8c13ff437a26ed
+size 426550448

modeling_minimax.py

@@ -44,7 +44,7 @@ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
 from transformers.processing_utils import Unpack
 from transformers.utils import TransformersKwargs, auto_docstring, can_return_tuple
 from transformers.utils.generic import OutputRecorder, check_model_inputs
-from .configuration_minimax import MiniMaxConfig
+from configuration_minimax import MiniMaxConfig
 
 
 @use_kernel_forward_from_hub("RMSNorm")