Upload tiny random Llama-3.3 model (seed 42)

README.md

@@ -16,16 +16,16 @@ This is a tiny random version of the meta-llama/Llama-3.3-70B-Instruct model, cr
 
 - **Base model**: meta-llama/Llama-3.3-70B-Instruct
 - **Seed**: 42
-- **Hidden size**: 768
+- **Hidden size**: 256
 - **Number of layers**: 12
-- **Number of attention heads**: 12
+- **Number of attention heads**: 4
 - **Vocabulary size**: 128256
-- **Max position embeddings**: 2048
+- **Max position embeddings**: 131072
 
 ## Parameters
 
-- **Total parameters**: ~272,517,888
-- **Trainable parameters**: ~272,517,888
+- **Total parameters**: ~75,110,656
+- **Trainable parameters**: ~75,110,656
 
 ## Usage
 

config.json

@@ -12,15 +12,15 @@
   ],
   "head_dim": 64,
   "hidden_act": "silu",
-  "hidden_size": 768,
+  "hidden_size": 256,
   "initializer_range": 0.02,
-  "intermediate_size": 2048,
-  "max_position_embeddings": 2048,
+  "intermediate_size": 768,
+  "max_position_embeddings": 131072,
   "mlp_bias": false,
   "model_type": "llama",
-  "num_attention_heads": 12,
+  "num_attention_heads": 4,
   "num_hidden_layers": 12,
-  "num_key_value_heads": 4,
+  "num_key_value_heads": 2,
   "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
   "rope_scaling": {

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:51111d7052986501f4f8976ed86e6c838a273ddea1d9d8ce12d92d05c4d75c80
-size 1090083944
+oid sha256:90be80471e3b86e6f1aaea7c053977f9aa0f720669ca9635b5f97bba661a0f2c
+size 300454912