arch-space commited on
Commit
fdba9fa
·
verified ·
1 Parent(s): 06f4649

Add files using upload-large-folder tool

Browse files
README.md ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: transformers
3
+ pipeline_tag: text-generation
4
+ tags:
5
+ - olmo3
6
+ - safetensors
7
+ - sliding-window-attention
8
+ - baseline
9
+ ---
10
+
11
+ # OLMo 3 3B Baseline — Stage 2 Mid-training
12
+
13
+ This repository is the Hugging Face export of `o3b3b-s2-s8192-g256-m1-ga1-tp2-cp1-dp256-h32-b8-mc2-lr2p071e4-512npu-share-0905081317-s2v1` at
14
+ iteration `47684`. This is the matched pure OLMo 3 baseline. It uses Transformers' official `Olmo3ForCausalLM` implementation and does not require remote code.
15
+
16
+ - Training sequence length: 8,192
17
+ - Model context capacity: 8,192
18
+ - Sliding-window size: 4,096
19
+ - Attention pattern: `[SWA, SWA, SWA, Full]`
20
+ - Vocabulary: 100,278 real tokens; 74 Megatron padding-only rows removed
21
+
22
+ Stage 3/4 use the frozen 65,536-token configuration. YaRN applies to the Full
23
+ Attention layers; SWA layers retain their original RoPE and 4,096-token local
24
+ window.
25
+
26
+ ## Loading
27
+
28
+ Use `transformers>=4.57.6,<5`.
29
+
30
+ ```python
31
+ import torch
32
+ from transformers import AutoModelForCausalLM, AutoTokenizer
33
+
34
+ repo_id = "ArchSpace-Collection/OLMo3-3B-stage2"
35
+ tokenizer = AutoTokenizer.from_pretrained(
36
+ repo_id,
37
+ use_fast=True,
38
+ fix_mistral_regex=False,
39
+ )
40
+ model = AutoModelForCausalLM.from_pretrained(
41
+ repo_id,
42
+ dtype=torch.bfloat16,
43
+ attn_implementation="sdpa",
44
+ )
45
+ ```
46
+
47
+ `fix_mistral_regex=False` preserves the exact tokenizer behavior used during
48
+ training. Conversion provenance, per-tensor hashes, and CPU validation results
49
+ are included in `conversion_manifest.json`, `SHA256SUMS`, and
50
+ `hf_validation_report.json`.
SHA256SUMS ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ 09758f58ce05f439ec1ccc73a188e654d5bed80de6a30560174c9e5f6c4e7023 README.md
2
+ 9b48c5952f5fbda254713d46cd8b64d47fdf2d3bf5e4c35b6b6d08c33dbb03df config.json
3
+ d1944b60fa9ca6d9dc19168562954296f4eccbebe798e2547765f88815992288 conversion_manifest.json
4
+ dbd4aa5dc6f7f1bdc0c181d9e5582f6571e1d2a22878dfcd98149324b9f5c4a2 generation_config.json
5
+ 2bb727f5fb155537394b6668dc0fe588842663edb1a38bb0ecc85c692da763fe hf_validation_report.json
6
+ b6b99b96496f96826d14c4b730e177268e1ffa0af4a6cd6d86549fb9f5f83bb1 model.safetensors
7
+ 4cb2b52960bababa8ec27153831e405ef7811e3fad1226106e09c28e715cfb21 special_tokens_map.json
8
+ 73fd5254624f39a88e3faac6a8e11300fc3c735ed37880d4f4f08db898eaecca tokenizer.json
9
+ df7285f1e002db7ce39f3da5b85abed05bbf8949e3f576051e60b51c73b64fdf tokenizer_config.json
10
+ 9e14712c91b37c7aab74b1306baa46ac342d620637a4b44523cdc3aec7d24195 vocab.json
config.json ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "",
3
+ "architectures": [
4
+ "Olmo3ForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": null,
9
+ "eos_token_id": 100257,
10
+ "hidden_act": "silu",
11
+ "hidden_dropout": 0.0,
12
+ "hidden_size": 3328,
13
+ "initializer_range": 0.02,
14
+ "intermediate_size": 13312,
15
+ "layer_types": [
16
+ "sliding_attention",
17
+ "sliding_attention",
18
+ "sliding_attention",
19
+ "full_attention",
20
+ "sliding_attention",
21
+ "sliding_attention",
22
+ "sliding_attention",
23
+ "full_attention",
24
+ "sliding_attention",
25
+ "sliding_attention",
26
+ "sliding_attention",
27
+ "full_attention",
28
+ "sliding_attention",
29
+ "sliding_attention",
30
+ "sliding_attention",
31
+ "full_attention"
32
+ ],
33
+ "max_position_embeddings": 8192,
34
+ "max_sequence_length": 8192,
35
+ "mlp_bias": false,
36
+ "model_type": "olmo3",
37
+ "num_attention_heads": 16,
38
+ "num_hidden_layers": 16,
39
+ "num_key_value_heads": 16,
40
+ "olmo3_weight_decay": true,
41
+ "pad_token_id": 100277,
42
+ "padded_vocab_size": 100352,
43
+ "qk_norm": true,
44
+ "qk_norm_mode": "full_projection",
45
+ "rms_norm_eps": 1e-06,
46
+ "rope_full_precision": true,
47
+ "rope_scaling": null,
48
+ "rope_theta": 500000.0,
49
+ "scale_embeds": false,
50
+ "scale_output_layer_init": false,
51
+ "sliding_window": 4096,
52
+ "tie_word_embeddings": false,
53
+ "torch_dtype": "bfloat16",
54
+ "transformers_version": "4.57.6",
55
+ "true_vocab_size": 100278,
56
+ "truncated_normal_factor": 3.0,
57
+ "use_cache": true,
58
+ "vocab_size": 100278,
59
+ "vocab_z_loss_coeff": 1e-05
60
+ }
conversion_manifest.json ADDED
@@ -0,0 +1,228 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema": "olmo3.base.hf_export/v1",
3
+ "source": {
4
+ "checkpoint_metadata_sha256": "6e58e0f612e993539725826584d8a290bef548b73f1f72095454c3af3254a7a0",
5
+ "iteration": 47684,
6
+ "model_schema_sha256": "fe3619e2a8baab27da73cae915e6ad17aa1853264f12965e058ecafef00249bc",
7
+ "parameter_count_with_padded_vocab": 3503508736,
8
+ "run_id": "o3b3b-s2-s8192-g256-m1-ga1-tp2-cp1-dp256-h32-b8-mc2-lr2p071e4-512npu-share-0905081317-s2v1"
9
+ },
10
+ "stage": "stage2",
11
+ "target": {
12
+ "format": "huggingface_olmo3_safetensors",
13
+ "padded_vocab_size": 100352,
14
+ "parameter_count": 3503016192,
15
+ "tensor_count": 179,
16
+ "tensor_sha256": {
17
+ "lm_head.weight": "21b1a588da9c549026f1c06fb8b44a690fa99e179b654a7a03786e4d1b969217",
18
+ "model.embed_tokens.weight": "f2509aa335a2ec46a98f898bcb5c27a33dfd1826441946e7b5ddf135db687f64",
19
+ "model.layers.0.mlp.down_proj.weight": "6b640939dbadf704dba81cc210b2c5dcf11428d33985a6464663055bb60a2c45",
20
+ "model.layers.0.mlp.gate_proj.weight": "576eda711e352de7348e81223b8205b01dc7bb7fdfb5ba1c89b35af7e8961226",
21
+ "model.layers.0.mlp.up_proj.weight": "98c500afe89f922495493f0c92363c6eaab4ef55e6641d31819e5f23c4395cc1",
22
+ "model.layers.0.post_attention_layernorm.weight": "01b721fce7c1b06203ce422f6a5f17331bd79b7f07b8269298e73df2fc69c3a6",
23
+ "model.layers.0.post_feedforward_layernorm.weight": "07e446320836cc4601cd1a953be6e080189b78158d75598883cf98e8a0170e48",
24
+ "model.layers.0.self_attn.k_norm.weight": "9f89bb728d22c6185133fcb1b5cbde6347552530b9cd14ca69ba8b92b1fb0aaa",
25
+ "model.layers.0.self_attn.k_proj.weight": "a62bd6a670cf5548bdc30a00fabd5402bb0e5304ac9e874c59e6be097b6b10f1",
26
+ "model.layers.0.self_attn.o_proj.weight": "e14ca813f44630da17f5f7a89553926311fe31e477d2f15e51e20d18f6a2fcf7",
27
+ "model.layers.0.self_attn.q_norm.weight": "2be468c2920cb1d87a395d599ef251dd7d7a7050a70bcbff8e023dfd71124b66",
28
+ "model.layers.0.self_attn.q_proj.weight": "264859710b5a469ca1dc18f54471fef7f90aae0058e021b002163aa699ee7bcd",
29
+ "model.layers.0.self_attn.v_proj.weight": "478978059a14d0fb6ead0ed4a4317ba28cf9e134405cc9b66ef7e07d09d5a3d9",
30
+ "model.layers.1.mlp.down_proj.weight": "95a7fb2e63103c8c7ccc255bccf677df031b53e3b5cb1d5a176ea189a03e4a0c",
31
+ "model.layers.1.mlp.gate_proj.weight": "e0cd7310758afcd711d2cc01f3a3b4b6ede9bae78b2dc0880c3e0f6861b6edfc",
32
+ "model.layers.1.mlp.up_proj.weight": "0561adb499ec192c7795ee767d8f883d893283973acf6b5fcd1daf4f3cc38654",
33
+ "model.layers.1.post_attention_layernorm.weight": "67dfd43efed4644bf33f80f286af57d22d511f3796085adccac16946387c9a7b",
34
+ "model.layers.1.post_feedforward_layernorm.weight": "d3585723a7126763fd67259cef682e31e0a919fd554baf0b7b3a32f972e6f9aa",
35
+ "model.layers.1.self_attn.k_norm.weight": "7eab5622f70375bf01d1815ae517993353c1409811d6292aeb82fe8bce986743",
36
+ "model.layers.1.self_attn.k_proj.weight": "3ebde5302dec1d3b32b0971a1ba7fd7b036af4796e4bb0c8ace085b99b1dc7cb",
37
+ "model.layers.1.self_attn.o_proj.weight": "304d098de7a1c43938f91a0c74722836b7e08ed0ea750e85952377af4261186e",
38
+ "model.layers.1.self_attn.q_norm.weight": "8ca4e1024cfa7e321bff1f2403d66cbbd00eeaf68f272834575bc20b0b9f0d07",
39
+ "model.layers.1.self_attn.q_proj.weight": "23e355e63ed9144f141d7f2518ce58fa60391c31fe385379a02fb88ed5a0ec38",
40
+ "model.layers.1.self_attn.v_proj.weight": "701207892a5db9ecf68a2aa8ebdcb2c3f0393cd9fae4c44a8dd50af836d286bd",
41
+ "model.layers.10.mlp.down_proj.weight": "e09f59e55a28ccd8ca7ab39e149cba8482ebec1e40eebfe6c9f7db5219d55aa7",
42
+ "model.layers.10.mlp.gate_proj.weight": "a520866613c5bbd1c4facd25217da50a89b4b0e0bb9da5f74f2bdbdb4cb7515c",
43
+ "model.layers.10.mlp.up_proj.weight": "619361c232ecf9f61175166613640f515d033b8795ccd18d64c68061618c827f",
44
+ "model.layers.10.post_attention_layernorm.weight": "ed9c6c6a46f25af1fd78d51c9e1e424629fcb896d2731e4c0e269a875d1eeb7b",
45
+ "model.layers.10.post_feedforward_layernorm.weight": "4cf6bacb17a15d1d2257616584ad9ea644a6cd3f029e4f2f6b24149f12df56d2",
46
+ "model.layers.10.self_attn.k_norm.weight": "d45d1cf935b8333ff9a6f7561091956819dcc48c0fb648e264880796c4a08003",
47
+ "model.layers.10.self_attn.k_proj.weight": "b60274d436a218756231d61c97798a78b6b2cfe6325375e35fe95c6b62a42666",
48
+ "model.layers.10.self_attn.o_proj.weight": "b0be309b956d2e1cff6b9a0351b5391fa3c673f36230f347eb6b736c945c851a",
49
+ "model.layers.10.self_attn.q_norm.weight": "84c3b017740b5299f2334d37e80aecf5544d4cfe354338aec1061b3ee0bba099",
50
+ "model.layers.10.self_attn.q_proj.weight": "eed6cf0779a9592a76c31f4e334489d643cdf38867ee77ef760a1bb9b27890bc",
51
+ "model.layers.10.self_attn.v_proj.weight": "16413c8eba29303bb1575b0ef0275d70d568e9f0295f09a185dfc7e036e68138",
52
+ "model.layers.11.mlp.down_proj.weight": "fe9260184f8ca58f928524d4e11ca2c7031da881b9bd998681d8c5548aaf8764",
53
+ "model.layers.11.mlp.gate_proj.weight": "e0c29926895bbb94ede5e3d1025750257657d50ea686773b2edf79f1ecd642fb",
54
+ "model.layers.11.mlp.up_proj.weight": "ae31c1f46c7af300690f8772ac1141151608d77a31053c98bcee293bddb56a8f",
55
+ "model.layers.11.post_attention_layernorm.weight": "f5153ccb18621d6078f4dc1dca76bc981025ff919929641007069d4b8387292a",
56
+ "model.layers.11.post_feedforward_layernorm.weight": "c6702a1b8485f64ce494c1273743078dc33130b78a18c8922324fba3ab10765e",
57
+ "model.layers.11.self_attn.k_norm.weight": "93b9c86ea7fe572182cb9c9a013e45e74af2e36d53ef40561e4a268252e12e63",
58
+ "model.layers.11.self_attn.k_proj.weight": "92ba6f6772e27153a6405f983cc12e0b0bbf0f58d062c73b666ff9869e37dcd9",
59
+ "model.layers.11.self_attn.o_proj.weight": "50c1a4309ccafd8ae36d8f36db1a3dc88f6c14b54a3889ea575b2f32b0035fed",
60
+ "model.layers.11.self_attn.q_norm.weight": "55c60a80d7d40b0f2447348d03bfeb1f6f6e277d552445dc16deae98fecfa54a",
61
+ "model.layers.11.self_attn.q_proj.weight": "3bc94cbe9ccb57618c93995545fbe500a3da160ae7aa1c85f53b10c69fc75237",
62
+ "model.layers.11.self_attn.v_proj.weight": "ea5b988654424f9979e1240ffee213359ab391a22438f0253fe7cefeeeb1a562",
63
+ "model.layers.12.mlp.down_proj.weight": "294e7a13f0f6e3d1378eb02ef341f284395cac4530b81283b0ea447bd997baa3",
64
+ "model.layers.12.mlp.gate_proj.weight": "08b14c8d096e87c9c5f25e1bc06db9a9bf80f0761ffc572502212e82100147f6",
65
+ "model.layers.12.mlp.up_proj.weight": "74bb411bf54704c9126b202d3639c1cdb7ae9cbb97000a85f624dc41465e776d",
66
+ "model.layers.12.post_attention_layernorm.weight": "9db71b1c78b7bd9590d2215fb66532ad1a59b866b2b074e4e5ea65ad501a4d6c",
67
+ "model.layers.12.post_feedforward_layernorm.weight": "df31293b17b0ce267e3a0054a7020f903a00ba0379fbb410119238f4314686d4",
68
+ "model.layers.12.self_attn.k_norm.weight": "f2e0946ba3213843bccb1a58b711b8fb75700c994635497000aae95369778c9f",
69
+ "model.layers.12.self_attn.k_proj.weight": "3d5936fa3f866d8167063adab49d3132e90a6caad89be4349a28412c0fa6a7ab",
70
+ "model.layers.12.self_attn.o_proj.weight": "18efabe16c405a2623583aba1f3e19356c15471f9d2ba8cd0be303a3f4c89a86",
71
+ "model.layers.12.self_attn.q_norm.weight": "e0516ad1877ee0b313f57b68cfa5b0b361773a074cb005911c60208cb0243724",
72
+ "model.layers.12.self_attn.q_proj.weight": "09f76a1dc97f4f7b628227258c93a2f0d7d614defb1ba7f5c79f951bf60d7dee",
73
+ "model.layers.12.self_attn.v_proj.weight": "a00850eec2eb9db32e2097d43c7ccdfa782bb9c9b196da50d3e899c98e5e6de2",
74
+ "model.layers.13.mlp.down_proj.weight": "926ef698a75de533e1caac6e8bc0c9ef58925cb2814871a5a6cf0a06190adac6",
75
+ "model.layers.13.mlp.gate_proj.weight": "a2ba090ba91d7e2977eda87fdf1f5f519ee75dc6df9ef8b08a1c6f2a37893cee",
76
+ "model.layers.13.mlp.up_proj.weight": "60271386144a5f4b7f24d5f90d62a16e879f8e063d0dd1f4fe889c3898a04c44",
77
+ "model.layers.13.post_attention_layernorm.weight": "38d13bff415e0d43142c059977d494737aadcb5b3b0211aa5e0f904a13120714",
78
+ "model.layers.13.post_feedforward_layernorm.weight": "45297e613978b0b9449b2869d2c4318a2bab2790faf1441afdcc23198c70034d",
79
+ "model.layers.13.self_attn.k_norm.weight": "3d29f44643b074a8161236b7389a1189ef4e448928a621a1cf9e6d8f180b9559",
80
+ "model.layers.13.self_attn.k_proj.weight": "da06a603b052785167597c5f7d4021a01fc1a424d02725507fd3cbb4c13034f2",
81
+ "model.layers.13.self_attn.o_proj.weight": "9950efe7fe009ef33a7ce6f07f6175ae84ec4b19f6c0ddc4c0b47ba6bf6c1eac",
82
+ "model.layers.13.self_attn.q_norm.weight": "4c7914278f686620aaf35ed2d76e09207e0c3bc0102eecc2a48ab428e7cb2667",
83
+ "model.layers.13.self_attn.q_proj.weight": "d3fd14193502b292162b2a696e5c7d1f6ec615a5905ffa2aeddb14b92146f9ae",
84
+ "model.layers.13.self_attn.v_proj.weight": "eb0ef30d5f6635477ccb8681a41038ab06629dbeba2c6ff8928583d87b3751c3",
85
+ "model.layers.14.mlp.down_proj.weight": "0a04c3e27183723ac36226734b0a1beecf1c27e8262590dc1bd1a9ebd5adb7f0",
86
+ "model.layers.14.mlp.gate_proj.weight": "10ee06800aa77bea262585435a9c51e7f81587f49ffb1300d23fc6d7852620f6",
87
+ "model.layers.14.mlp.up_proj.weight": "2e4fd1cc3f89d0bcb4cd500ea8096d95977d464a55b1cde3e08c0c9e438087e8",
88
+ "model.layers.14.post_attention_layernorm.weight": "292eb8a5b22a6bc7d956766612de0190d964cd88dd2f4cd9af3e08a2a3af2aa5",
89
+ "model.layers.14.post_feedforward_layernorm.weight": "1605168d60e4982665b6fd2ed237d6544985bbb45fa10a4ea237958fd306b62e",
90
+ "model.layers.14.self_attn.k_norm.weight": "05c43339155dd467d39caa8c216c5cef28d80f16b09dbafe70041dc18c1514a2",
91
+ "model.layers.14.self_attn.k_proj.weight": "b65937adda145580089cd7eafc742ef8abee3f06ef4d8b3dc13cb9123ce06539",
92
+ "model.layers.14.self_attn.o_proj.weight": "a8428d7ff146118b9620448989ba9d0283c88a859d62efc0e4962758f38157c1",
93
+ "model.layers.14.self_attn.q_norm.weight": "213918564fcfb37a915484e6f447cee462fce1a1e795292f19cc7d9eec640de9",
94
+ "model.layers.14.self_attn.q_proj.weight": "a12df8b438a943bb60a96d2ad2d2b033c358317bf38bbd360664c94cdb1e8f7c",
95
+ "model.layers.14.self_attn.v_proj.weight": "bd0928845131d8a815f94d4ecc4a10c201a1664b34715496e619d43c6af4cda9",
96
+ "model.layers.15.mlp.down_proj.weight": "31b7d7018c8a7f312db880c7b3794698369664d5aec5d6a8d1ea46e05f7fdd86",
97
+ "model.layers.15.mlp.gate_proj.weight": "7d060a5c03a2265b362f652791a44719d0524f5b8f220ba5ece0de0dcf60d114",
98
+ "model.layers.15.mlp.up_proj.weight": "3bc4f9f5390def0829fa52f36ab334c8f634a91a5fb5cf73786f094e7def7dbc",
99
+ "model.layers.15.post_attention_layernorm.weight": "9e289e1414b9827002a189ce96505b5edd366ebe5700791d0e390eed99651acb",
100
+ "model.layers.15.post_feedforward_layernorm.weight": "c8f52bec2a0868270e4f2b5e90f545c3b0e0008b07b425ffa4a71ea06b64ade9",
101
+ "model.layers.15.self_attn.k_norm.weight": "91a0bc8782a69840fb340a81525cfcfcd34c9fdf3a90ac71ace601fb0ea3214e",
102
+ "model.layers.15.self_attn.k_proj.weight": "44aa75877675d27c7c1861866041babaf4a5a79c55dd28806893a5283a63adf0",
103
+ "model.layers.15.self_attn.o_proj.weight": "eb204f119ba7c826df7a8e56107b4c4491df947a398c6f391359525b0796f360",
104
+ "model.layers.15.self_attn.q_norm.weight": "62267fc8ef6a409e827ef90449a3e48681458823da35262382e89f5bdc1ca856",
105
+ "model.layers.15.self_attn.q_proj.weight": "5ca68150ca7c5f5f7e790977f89972f9675aa61cb7d247a4dbebb8fed28869f4",
106
+ "model.layers.15.self_attn.v_proj.weight": "1a6ca3e3e910a9682d8a13e713fa41796e36ad87356923210ee39f78104d0f17",
107
+ "model.layers.2.mlp.down_proj.weight": "19a3eb2e468549a8d0fa553488051b7b503e14a9f404e0394fa91145f52e2012",
108
+ "model.layers.2.mlp.gate_proj.weight": "6712d2475f4cf631a27ba56d859e407dc2f5132df1cd21fa1b9a939e30040ff2",
109
+ "model.layers.2.mlp.up_proj.weight": "0dfd6aba9169bdf02d2f52ade37ed05c57f68386c3639789a886e0a9995c0683",
110
+ "model.layers.2.post_attention_layernorm.weight": "e64634e44362c6916bc5c1ff80c9a11655e0d97ef690a66b371886f4b8552e54",
111
+ "model.layers.2.post_feedforward_layernorm.weight": "ceb8b351a7223d21d95072dcd69c81a5046a51f7898d037a85835a53814dd6ac",
112
+ "model.layers.2.self_attn.k_norm.weight": "3e93c5855a87b3dc25c85ce495a48120693bdc0d9c67b9aba9985d5189fab2bc",
113
+ "model.layers.2.self_attn.k_proj.weight": "40990b734f71cde5db74318919446531f3539745cc4100ada54bd3314d259e0b",
114
+ "model.layers.2.self_attn.o_proj.weight": "d2e017b07844aa9292af6460df93492c08ac3ee5542ebb79bffe7aeccb1591a1",
115
+ "model.layers.2.self_attn.q_norm.weight": "eba198aae72f0f45a7f04361979b068c1a7c6f44484c411e9b609d5a7e058a2b",
116
+ "model.layers.2.self_attn.q_proj.weight": "b774cd49127175422f9d3a1b80f0192224c17f0231fa60ae2bf408a1160b08a2",
117
+ "model.layers.2.self_attn.v_proj.weight": "6c752908d81fa467e1fc6a4d38d2b8b345069e5af24f34a2a701b2fdca157ec6",
118
+ "model.layers.3.mlp.down_proj.weight": "bce7b4ccd955156690f704c84467942b9e117c09f35a3b1e5a5f1cc8e8c37235",
119
+ "model.layers.3.mlp.gate_proj.weight": "20e1df02cf8ceaa96979799189b911f36ea4e8cb013d93dcaa0a8c25a319a910",
120
+ "model.layers.3.mlp.up_proj.weight": "f0350848f77462725cd4f95ff7edd141c1a1b73a2a714992131458971558fd95",
121
+ "model.layers.3.post_attention_layernorm.weight": "9836f40bad938939617718699143545fe33a54407fabd0b056fc6ce3955cc552",
122
+ "model.layers.3.post_feedforward_layernorm.weight": "6ee146253b594a4889ce775b48faf01cbe4d4c5508cbcbf72a0c29455050bd4a",
123
+ "model.layers.3.self_attn.k_norm.weight": "d4a964b6b533436ab6b5c119f72e34b14f7f95c918bdc35860b7324f47dcbb4f",
124
+ "model.layers.3.self_attn.k_proj.weight": "881a892bc82c64614ecae80496daf3e682c54847dda6f8c656087c3100255bc1",
125
+ "model.layers.3.self_attn.o_proj.weight": "cf1a835dfc3a38d5ee2729aa671be483bb7ab5f4bc3f6954a5da0752511f91a2",
126
+ "model.layers.3.self_attn.q_norm.weight": "7fa3ac6d573d89a870b191ab4a7db17f16ca008320d221b0a189604ad238d25c",
127
+ "model.layers.3.self_attn.q_proj.weight": "81cac064a99ab6bb3bf41f5205e38dab00bb2369d886d210367a4ca03393ba5d",
128
+ "model.layers.3.self_attn.v_proj.weight": "282386b393c6c0fea0f8bdd02d119e7e42c90f233edfd27da230309542083adf",
129
+ "model.layers.4.mlp.down_proj.weight": "60dd7da48130441acec04971b4ecbc0c84cce69ccb8f493d39e5dbe046ea7a40",
130
+ "model.layers.4.mlp.gate_proj.weight": "a98c9429c71c8c01c1fbdfc297de5e7513de776082f414012fc403c11edc116a",
131
+ "model.layers.4.mlp.up_proj.weight": "feb2b417f957dd0b9804528d6da3fad420fc15e3358b80448f11ba78f67530cc",
132
+ "model.layers.4.post_attention_layernorm.weight": "b1cb99eda4dd0dde805fa184eee6304436a7edf65ab7cf249e7132ba892447a3",
133
+ "model.layers.4.post_feedforward_layernorm.weight": "0da76649262a4741b367384c38bc4b646b49304e835971484c09a77126d794f4",
134
+ "model.layers.4.self_attn.k_norm.weight": "bd3dffb8c1ffd9710385f8aa53e8d03b76f904e6ca05173080dd1ae39ea0f84b",
135
+ "model.layers.4.self_attn.k_proj.weight": "a678b41cdb872c804f26fc9cb1719d34338de861acff015d54bf252aa4e4cdf3",
136
+ "model.layers.4.self_attn.o_proj.weight": "58dcb2704e8724a33b6e3f86931c2e0da3e5eba0f08d79fffa1ea8337e473856",
137
+ "model.layers.4.self_attn.q_norm.weight": "10a5ff9388feaf68e4ca0006751304d6f7a47c9b7d5c0a1501883350798e4eb4",
138
+ "model.layers.4.self_attn.q_proj.weight": "2ae7f826f6e314a1648c19bb02414ac7ee596c48fcdd748b662e00eaac673640",
139
+ "model.layers.4.self_attn.v_proj.weight": "5c3260bd6fd01c7bc7f3694c23b4249918323d8ee6be06d97e0d7d1a896a3464",
140
+ "model.layers.5.mlp.down_proj.weight": "04588b9aac4788a7fed31682e79aedb0d6aac6d9638ca11f7b93f27e7b25c411",
141
+ "model.layers.5.mlp.gate_proj.weight": "2bdf275f820d284cb0cf207acc75cf0a881fca5d02f4f5e1643a21d0358cef60",
142
+ "model.layers.5.mlp.up_proj.weight": "1435dd5f87216c92d41e03b850d443976f17f5c4619433712e1cb932811b7679",
143
+ "model.layers.5.post_attention_layernorm.weight": "45fd23073d324f7c57e93b60ff2d75094fe58d1a201a198f47824f16643898e3",
144
+ "model.layers.5.post_feedforward_layernorm.weight": "79e8b77a2c847f42584a13a843b93f69eafedcf15e81b0c5050ca48eca928aa6",
145
+ "model.layers.5.self_attn.k_norm.weight": "0102da63eaa5e41d7b3b177330a6dc64eff71d9e794ec9719c0dcdba4ad779e4",
146
+ "model.layers.5.self_attn.k_proj.weight": "8269186cf34054aad217bd1dd3ad38dd14f9c36c9c729ee9572a88749d883f37",
147
+ "model.layers.5.self_attn.o_proj.weight": "93493f79e5fa58f35e96c2b4aee31c86eafa61d8fc2567c3bb35f6d7efc62d2d",
148
+ "model.layers.5.self_attn.q_norm.weight": "14470322a5812a35d789551db19f9b815aa444198eb7ca067c013829964fb92d",
149
+ "model.layers.5.self_attn.q_proj.weight": "91d04a7dc9253598016caf1656a2a28cc55882e354f09bbc532cb56f30f960cc",
150
+ "model.layers.5.self_attn.v_proj.weight": "a96f4bcbbb733a17d4157216dc7baf0b9e531465525a2e6b2d86ef10e93dc021",
151
+ "model.layers.6.mlp.down_proj.weight": "c4fe6e49c150a9b76f548788df7f102fe564a6e5884031b2fe3ff1b8df7595cb",
152
+ "model.layers.6.mlp.gate_proj.weight": "2ae0f3babe7efaae7c1c7129aac92d615c7f4138c467e47f54633589dcef6589",
153
+ "model.layers.6.mlp.up_proj.weight": "c2322512a43535e31b541ab889e6dfba0ed0e862d68120e69ce090609fccb275",
154
+ "model.layers.6.post_attention_layernorm.weight": "2a12acd4f549073a62f775201e47040b7a4317b9c0087596d5b8061dff2a9704",
155
+ "model.layers.6.post_feedforward_layernorm.weight": "5b98b456281045a0d7e7f4e0d52446e0d0ca3851f565c4b4d986cdd3a465950d",
156
+ "model.layers.6.self_attn.k_norm.weight": "b24b8af9b6e6ff179dacab196e1c1b0882b3c8dfe4be9858028c7a4260a1494c",
157
+ "model.layers.6.self_attn.k_proj.weight": "cf7b04f58431e106342c6f29138fc707630d6286f12f55d29acb73b61fd669e4",
158
+ "model.layers.6.self_attn.o_proj.weight": "e019f187a9e6cd174979d711c38146a32fb6307b0551f072c498153244a728c0",
159
+ "model.layers.6.self_attn.q_norm.weight": "8ea5ee02e1d8c004eef21f6d42be397b4f8f359b9907095341723208ce9eed11",
160
+ "model.layers.6.self_attn.q_proj.weight": "801c2c6f122dd5817f4c9626231b606607972d184e9ed425a24f02bc4dc70f44",
161
+ "model.layers.6.self_attn.v_proj.weight": "53be243084a22fc0bb4382ee4e7b91435b117562de857dbb4e98475439939d01",
162
+ "model.layers.7.mlp.down_proj.weight": "2052df6fc325c7536168cdac07c5d64cff08b918994665f565e98a9708956101",
163
+ "model.layers.7.mlp.gate_proj.weight": "28d465666075cd42e73b12b009a33ec6e587b8ad43079f5ed01963f9a091b5f0",
164
+ "model.layers.7.mlp.up_proj.weight": "cf659742c4bd7d535b77a5f765aaa6a7178439c2f3995dd6f8dd52b25b775e27",
165
+ "model.layers.7.post_attention_layernorm.weight": "78634f53afdcde2cc255a9e88aff08161dd7d660c14e55fa4454dbc105b65ef8",
166
+ "model.layers.7.post_feedforward_layernorm.weight": "e04130809ceef94cddb0fc2a4a5257d1a21de8325704b22b69c9664167107b62",
167
+ "model.layers.7.self_attn.k_norm.weight": "55b297c74bd06e0f69d58a3e15feeeee56ee356ea8894defa3f680e935665c0a",
168
+ "model.layers.7.self_attn.k_proj.weight": "5a65f445b2139e7d793b6d6952f5a9e1ba4b603ce709de82b1483c1a2ccaadc9",
169
+ "model.layers.7.self_attn.o_proj.weight": "3d386fe7339e1f7bcf3409588542869fcb5b2f38de9a78ffc204d8e1eac1ea92",
170
+ "model.layers.7.self_attn.q_norm.weight": "0bb020cf2bc913919c0728638b94ef4d1b291e713dfd9c427c5ec258358f8ea4",
171
+ "model.layers.7.self_attn.q_proj.weight": "47c809422d70ff8c3a1f251c9a32702598e04f03fe4466986d8dba014e5b71e6",
172
+ "model.layers.7.self_attn.v_proj.weight": "d9e3066ef1a8441f441f86f1363888963dd400aebab5e29d10c176ac0206e2e6",
173
+ "model.layers.8.mlp.down_proj.weight": "c37a9bd3da62f526de6b4dd91130a14811d0a988a40919e9476af60bfdedda6c",
174
+ "model.layers.8.mlp.gate_proj.weight": "2f1209ced00158e71dc80c29be9e784bf8ccaad7f847e4bef7f03bbd78170176",
175
+ "model.layers.8.mlp.up_proj.weight": "7a0d8a1c0f6fc5309b0b1cbd66038d47301082aa95d85f38a930647d613eb921",
176
+ "model.layers.8.post_attention_layernorm.weight": "8fbbba5c8dd2ad0946140486f631528f8b6edf1e00eae0b1f0b141a86bb63c50",
177
+ "model.layers.8.post_feedforward_layernorm.weight": "8286dd608dfe70340c72be6207da82d2ca25733ad746e37a6b2693b5ecaa1e3b",
178
+ "model.layers.8.self_attn.k_norm.weight": "d10a9a89bd1b78b657ec99bfb793f77012bf3de5f2a9555268f8abb48eac73c3",
179
+ "model.layers.8.self_attn.k_proj.weight": "5d90ea30a4b0398136799fd278676ee9e5467403ed73332949bc01922d5f5180",
180
+ "model.layers.8.self_attn.o_proj.weight": "5ecb19eddace4e32319be933e141964ca8903b79b10634c9e82a682a4cd8c079",
181
+ "model.layers.8.self_attn.q_norm.weight": "b8d7d95b9f980ff305dcddc2f8d8082c1b8c1fb0e6a59afb0e81d6e6884d74c2",
182
+ "model.layers.8.self_attn.q_proj.weight": "62111098bdb3acb8bb0ea94a29400dff93752f8ce7f805fe85f230170ffca9de",
183
+ "model.layers.8.self_attn.v_proj.weight": "7d2518f441b94e58c42eb4c0e78db1717a52bed124d4bc817dd84c376cf62b45",
184
+ "model.layers.9.mlp.down_proj.weight": "1007053d618caf76729761957c9a37d8d060fe497b0b22484e35b8a5775fee39",
185
+ "model.layers.9.mlp.gate_proj.weight": "1426fce091c93ce64fbe672eb5cea131b21225fc3da94b2a50164bb55488ebce",
186
+ "model.layers.9.mlp.up_proj.weight": "0ebc714880ca8a6acc721251a15b3be810e40d138725a8998088ffb00e68e53c",
187
+ "model.layers.9.post_attention_layernorm.weight": "1ef9bfa6ed0ab825cdcf9952f9b2e7cc16e1ae0ab4fff362cf24acdc80a83c0d",
188
+ "model.layers.9.post_feedforward_layernorm.weight": "261d9f071d95a871a232835edfc271b4d9bfb55258036e76b62a1fc25bea6923",
189
+ "model.layers.9.self_attn.k_norm.weight": "805d930822001277d6151b2c8ca33413454beae3377953c293a6317f0377bd08",
190
+ "model.layers.9.self_attn.k_proj.weight": "2d94f43b0bc6e71630c7159c410bfa05ebfb23536d832b565299f0007934e596",
191
+ "model.layers.9.self_attn.o_proj.weight": "b6ae3874f7e4c037c4e940c512859e20c84aba10540d3c84fd97a53caf45e0f0",
192
+ "model.layers.9.self_attn.q_norm.weight": "87b254f77de8106744b68e4a1f69404cf8dc0c3e6447e6c27eae69009b040e88",
193
+ "model.layers.9.self_attn.q_proj.weight": "c1560705c20ba7e2c175283613b7c91a2ab78f9f9104d8ffbcfef77d18e55586",
194
+ "model.layers.9.self_attn.v_proj.weight": "3a30d0273c5546d14cd5bcea200a54f7607e877883d190b7c0528b34cd1698e7",
195
+ "model.norm.weight": "22becf235261dbef24644a8e4cba43a477556bfc02cb8b56789e3ef455f101b6"
196
+ },
197
+ "tokenizer_files": [
198
+ "tokenizer.json",
199
+ "tokenizer_config.json",
200
+ "special_tokens_map.json",
201
+ "vocab.json"
202
+ ],
203
+ "transformers": ">=4.57.6,<5",
204
+ "true_vocab_size": 100278,
205
+ "vocab_crop": {
206
+ "embedding": {
207
+ "exported_rows": 100278,
208
+ "removed_rows": 74,
209
+ "removed_tail_sha256": "a386af5cc268529ee2bcfec1a9083d88a8dd46b746e452ba95477ec865f2eb1b",
210
+ "source_rows": 100352
211
+ },
212
+ "lm_head": {
213
+ "exported_rows": 100278,
214
+ "removed_rows": 74,
215
+ "removed_tail_sha256": "7b5a85862165e225fff3b126b0f4f01f22392c2865cd0fb1828097ab52de1623",
216
+ "source_rows": 100352
217
+ }
218
+ }
219
+ },
220
+ "validation": {
221
+ "fc1_split_repack": "bitwise",
222
+ "finite_model_tensors": true,
223
+ "qkv_split_repack": "bitwise",
224
+ "safetensors_reload": "bitwise",
225
+ "source_key_coverage": "exact",
226
+ "source_shapes": "exact"
227
+ }
228
+ }
generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": null,
4
+ "eos_token_id": 100257,
5
+ "pad_token_id": 100277,
6
+ "transformers_version": "4.57.6"
7
+ }
hf_validation_report.json ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checksummed_files": 9,
3
+ "forward": {
4
+ "cache_no_cache_max_abs": 0.0,
5
+ "logits_finite": true,
6
+ "logits_shape": [
7
+ 1,
8
+ 4,
9
+ 100278
10
+ ],
11
+ "parameter_count": 3503016192,
12
+ "post_8192_yarn_finite": null
13
+ },
14
+ "model_dir": "/data/PonderLM/fcsong/olmo3_hf_remote_exports_3b_20260907_v1/olmo3-base-3b-stage2-step47684",
15
+ "parameter_count": 3503016192,
16
+ "schema": "olmo3.3b.hf_validation/v1",
17
+ "stage": "stage2",
18
+ "status": "pass",
19
+ "tensor_count": 179,
20
+ "tokenizer_size": 100278,
21
+ "torch_version": "2.6.0+cpu",
22
+ "transformers_version": "4.57.6",
23
+ "variant": "baseline",
24
+ "weight_files": [
25
+ "model.safetensors"
26
+ ]
27
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b6b99b96496f96826d14c4b730e177268e1ffa0af4a6cd6d86549fb9f5f83bb1
3
+ size 7006053208
special_tokens_map.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "eos_token": "<|endoftext|>",
3
+ "pad_token": "<|pad|>",
4
+ "unk_token": {
5
+ "content": "<|endoftext|>",
6
+ "lstrip": false,
7
+ "normalized": false,
8
+ "rstrip": false,
9
+ "single_word": false
10
+ }
11
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,190 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "added_tokens_decoder": {
4
+ "100256": {
5
+ "content": "<|extra_id_0|>",
6
+ "lstrip": false,
7
+ "normalized": false,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": false
11
+ },
12
+ "100257": {
13
+ "content": "<|endoftext|>",
14
+ "lstrip": false,
15
+ "normalized": false,
16
+ "rstrip": false,
17
+ "single_word": false,
18
+ "special": true
19
+ },
20
+ "100258": {
21
+ "content": "<|fim_prefix|>",
22
+ "lstrip": false,
23
+ "normalized": false,
24
+ "rstrip": false,
25
+ "single_word": false,
26
+ "special": true
27
+ },
28
+ "100259": {
29
+ "content": "<|fim_middle|>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false,
34
+ "special": true
35
+ },
36
+ "100260": {
37
+ "content": "<|fim_suffix|>",
38
+ "lstrip": false,
39
+ "normalized": false,
40
+ "rstrip": false,
41
+ "single_word": false,
42
+ "special": true
43
+ },
44
+ "100261": {
45
+ "content": "|||PHONE_NUMBER|||",
46
+ "lstrip": false,
47
+ "normalized": false,
48
+ "rstrip": false,
49
+ "single_word": false,
50
+ "special": false
51
+ },
52
+ "100262": {
53
+ "content": "|||EMAIL_ADDRESS|||",
54
+ "lstrip": false,
55
+ "normalized": false,
56
+ "rstrip": false,
57
+ "single_word": false,
58
+ "special": false
59
+ },
60
+ "100263": {
61
+ "content": "|||IP_ADDRESS|||",
62
+ "lstrip": false,
63
+ "normalized": false,
64
+ "rstrip": false,
65
+ "single_word": false,
66
+ "special": false
67
+ },
68
+ "100264": {
69
+ "content": "<|im_start|>",
70
+ "lstrip": false,
71
+ "normalized": false,
72
+ "rstrip": false,
73
+ "single_word": false,
74
+ "special": true
75
+ },
76
+ "100265": {
77
+ "content": "<|im_end|>",
78
+ "lstrip": false,
79
+ "normalized": false,
80
+ "rstrip": false,
81
+ "single_word": false,
82
+ "special": true
83
+ },
84
+ "100266": {
85
+ "content": "<|extra_id_1|>",
86
+ "lstrip": false,
87
+ "normalized": false,
88
+ "rstrip": false,
89
+ "single_word": false,
90
+ "special": false
91
+ },
92
+ "100267": {
93
+ "content": "<|extra_id_2|>",
94
+ "lstrip": false,
95
+ "normalized": false,
96
+ "rstrip": false,
97
+ "single_word": false,
98
+ "special": false
99
+ },
100
+ "100268": {
101
+ "content": "<|extra_id_3|>",
102
+ "lstrip": false,
103
+ "normalized": false,
104
+ "rstrip": false,
105
+ "single_word": false,
106
+ "special": false
107
+ },
108
+ "100269": {
109
+ "content": "<|extra_id_4|>",
110
+ "lstrip": false,
111
+ "normalized": false,
112
+ "rstrip": false,
113
+ "single_word": false,
114
+ "special": false
115
+ },
116
+ "100270": {
117
+ "content": "<|extra_id_5|>",
118
+ "lstrip": false,
119
+ "normalized": false,
120
+ "rstrip": false,
121
+ "single_word": false,
122
+ "special": false
123
+ },
124
+ "100271": {
125
+ "content": "<|extra_id_6|>",
126
+ "lstrip": false,
127
+ "normalized": false,
128
+ "rstrip": false,
129
+ "single_word": false,
130
+ "special": false
131
+ },
132
+ "100272": {
133
+ "content": "<|extra_id_7|>",
134
+ "lstrip": false,
135
+ "normalized": false,
136
+ "rstrip": false,
137
+ "single_word": false,
138
+ "special": false
139
+ },
140
+ "100273": {
141
+ "content": "<|extra_id_8|>",
142
+ "lstrip": false,
143
+ "normalized": false,
144
+ "rstrip": false,
145
+ "single_word": false,
146
+ "special": false
147
+ },
148
+ "100274": {
149
+ "content": "<|extra_id_9|>",
150
+ "lstrip": false,
151
+ "normalized": false,
152
+ "rstrip": false,
153
+ "single_word": false,
154
+ "special": false
155
+ },
156
+ "100275": {
157
+ "content": "<|extra_id_10|>",
158
+ "lstrip": false,
159
+ "normalized": false,
160
+ "rstrip": false,
161
+ "single_word": false,
162
+ "special": false
163
+ },
164
+ "100276": {
165
+ "content": "<|endofprompt|>",
166
+ "lstrip": false,
167
+ "normalized": false,
168
+ "rstrip": false,
169
+ "single_word": false,
170
+ "special": true
171
+ },
172
+ "100277": {
173
+ "content": "<|pad|>",
174
+ "lstrip": false,
175
+ "normalized": false,
176
+ "rstrip": false,
177
+ "single_word": false,
178
+ "special": true
179
+ }
180
+ },
181
+ "bos_token": null,
182
+ "clean_up_tokenization_spaces": false,
183
+ "eos_token": "<|endoftext|>",
184
+ "extra_special_tokens": {},
185
+ "fix_mistral_regex": false,
186
+ "model_max_length": 65536,
187
+ "pad_token": "<|pad|>",
188
+ "tokenizer_class": "GPT2Tokenizer",
189
+ "unk_token": "<|endoftext|>"
190
+ }
vocab.json ADDED
The diff for this file is too large to render. See raw diff