llm001 L04 Continue

README.md

@@ -0,0 +1,28 @@
+---
+base_model: jodiox/olmo3-190m-zh-full
+license: apache-2.0
+language:
+- zh
+tags:
+- llm001
+- olmo3
+- chinese
+- continued-pretraining
+---
+
+# jodiox/olmo3-190m-zh-full-continue
+
+持续预训练版本：基于 jodiox/olmo3-190m-zh-full，
+在 cmz1024/llm101-olmo3-zh-demo-data/continue 数据上继续训练。
+
+## 训练配置
+- 数据：cmz1024/llm101-olmo3-zh-demo-data/continue/full_v31.bin
+- LR：2e-4（低 LR 防止灾难性遗忘）
+- Warmup：10%
+
+## 用法
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+model = AutoModelForCausalLM.from_pretrained("jodiox/olmo3-190m-zh-full-continue")
+tok = AutoTokenizer.from_pretrained("jodiox/olmo3-190m-zh-full-continue")
+```

config.json

@@ -0,0 +1,44 @@
+{
+  "architectures": [
+    "Olmo3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 2,
+  "dtype": "bfloat16",
+  "eos_token_id": 0,
+  "hidden_act": "silu",
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_types": [
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 4096,
+  "model_type": "olmo3",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "num_key_value_heads": 12,
+  "pad_token_id": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 500000,
+    "rope_type": "default"
+  },
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.7.0",
+  "use_cache": false,
+  "vocab_size": 48000
+}

generation_config.json

@@ -0,0 +1,12 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": [
+    0
+  ],
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 1,
+  "transformers_version": "5.7.0",
+  "use_cache": true
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e4e7c128a3f71d53c2d5e5895933a17460f94279d7fa2f3ba4e3d77538f62e1
+size 374038864

tokenizer_config.json

@@ -0,0 +1,12 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|bos|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 4096,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<|unk|>"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f169b07007537609082901be7d085fee91c73bb725ab300982c8e1549f629e7f
+size 4856