mohammadmahdinouri
/

mol-tiny-init-final

Model card Files Files and versions

mohammadmahdinouri commited on Oct 9, 2025

Commit

d4c7544

·

verified ·

1 Parent(s): aa1cd17

Upload DistillationWrapper

Files changed (2) hide show

config.json +3 -3
model.safetensors +2 -2

config.json CHANGED Viewed

@@ -8,7 +8,7 @@
   "dtype": "float32",
   "embedding_size": 128,
   "expert_intermediate_size": 2624,
-  "group_depth": 4,
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
   "hidden_size": 768,
@@ -21,13 +21,13 @@
   "max_position_embeddings": 8192,
   "model_type": "ModernALBERT",
   "num_attention_heads": 16,
-  "num_expert_modules": 2,
   "num_experts": 8,
   "num_hidden_layers": 12,
   "pad_token_id": 0,
   "router_jitter_noise": 0.01,
   "top_k": 2,
-  "transformers_version": "4.56.2",
   "use_adapter": true,
   "use_cache": true,
   "use_moa": true,

   "dtype": "float32",
   "embedding_size": 128,
   "expert_intermediate_size": 2624,
+  "group_depth": 3,
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
   "hidden_size": 768,
   "max_position_embeddings": 8192,
   "model_type": "ModernALBERT",
   "num_attention_heads": 16,
+  "num_expert_modules": 3,
   "num_experts": 8,
   "num_hidden_layers": 12,
   "pad_token_id": 0,
   "router_jitter_noise": 0.01,
   "top_k": 2,
+  "transformers_version": "4.57.0",
   "use_adapter": true,
   "use_cache": true,
   "use_moa": true,

model.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4cc8411a24b160e69b851a7e03285c4587ddc80a4c800460e21b604330bfa66b
-size 128468328

 version https://git-lfs.github.com/spec/v1
+oid sha256:d1670ae18ca7a2c98399a28649b1a3a833c651819e54a64f537c4430206d6cfc
+size 168188768