Upload 7 files

README.md

@@ -1,3 +1,58 @@
 ---
-license: mit
+language:
+- en
+license: apache-2.0
+tags:
+- spm
+- foundation-model
+- causal-lm
+- from-scratch
+model-index:
+- name: SPM-50M-Alpha
+  results: []
 ---
+
+# SPM-50M-Alpha
+
+**SPM-50M-Alpha** is the first model in the SPM (Strategic Pretrained Model) family.
+A ~71M parameter English foundation model trained from scratch.
+
+## Model Architecture
+
+| Parameter | Value |
+|-----------|-------|
+| Architecture | Decoder-only Transformer (Llama-style) |
+| Parameters | ~71M |
+| Hidden Size | 512 |
+| Layers | 12 |
+| Attention Heads | 8 |
+| KV Heads | 4 |
+| Vocab Size | 32000 |
+| Max Sequence Length | 2048 |
+| Positional Encoding | RoPE |
+| Normalization | RMSNorm |
+| Activation | SwiGLU |
+
+## Training
+
+- **Dataset**: FineWeb (sample-10BT)
+- **Tokenizer**: SentencePiece BPE (vocab=32000)
+- **Precision**: BF16
+- **Effective Batch Size**: 128
+
+## Usage
+
+```python
+from transformers import LlamaTokenizer, LlamaForCausalLM
+
+model = LlamaForCausalLM.from_pretrained("SPM-50M-Alpha")
+tokenizer = LlamaTokenizer.from_pretrained("SPM-50M-Alpha")
+
+inputs = tokenizer("The meaning of life is", return_tensors="pt")
+outputs = model.generate(**inputs, max_new_tokens=50)
+print(tokenizer.decode(outputs[0]))
+```
+
+## License
+
+Apache 2.0

config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "SPM-50M-Alpha",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "model_type": "llama",
+  "bos_token_id": 2,
+  "eos_token_id": 3,
+  "pad_token_id": 0,
+  "hidden_size": 512,
+  "intermediate_size": 1536,
+  "num_hidden_layers": 12,
+  "num_attention_heads": 8,
+  "num_key_value_heads": 4,
+  "max_position_embeddings": 2048,
+  "vocab_size": 32000,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "hidden_act": "silu",
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "initializer_range": 0.02,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.40.0",
+  "spm_version": "0.1.0"
+}

generation_config.json

@@ -0,0 +1,11 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": 3,
+  "pad_token_id": 0,
+  "max_new_tokens": 512,
+  "do_sample": true,
+  "temperature": 0.8,
+  "top_p": 0.9,
+  "repetition_penalty": 1.1
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0861bd629d000386c3cf0db0462c402bc455d05c5cde60f1ce82d88fbf4fd63e
+size 282132464

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "single_word": false,
+    "lstrip": false,
+    "rstrip": false,
+    "normalized": true
+  },
+  "eos_token": {
+    "content": "</s>",
+    "single_word": false,
+    "lstrip": false,
+    "rstrip": false,
+    "normalized": true
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "single_word": false,
+    "lstrip": false,
+    "rstrip": false,
+    "normalized": true
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "single_word": false,
+    "lstrip": false,
+    "rstrip": false,
+    "normalized": true
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce5baec7c99a91c46c3a1cd03e5d92457a9c1aefdef12ed34bcb51e6cc934c18
+size 794363

tokenizer_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "unk_token": "<unk>",
+  "pad_token": "<pad>",
+  "model_max_length": 2048,
+  "tokenizer_class": "LlamaTokenizer",
+  "legacy": false,
+  "sp_model_kwargs": {},
+  "add_bos_token": true,
+  "add_eos_token": false
+}