Hugging Face's logo

wytro
/
Know-Your-Sources-init

Model card Files
xet
 Community
wytro commited on
Commit
6fe2611
·
verified ·
1 Parent(s): 0afa3d6

Add files using upload-large-folder tool

Browse files
Files changed (50) hide show
  1. README.md +7 -0
  2. _init_1.5B_seed42/0/checkpoint_metadata.json +22 -0
  3. _init_1.5B_seed42/0/config.yaml +121 -0
  4. _init_1.5B_seed42/0/model/model/decoder/16/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  5. _init_1.5B_seed42/0/model/model/decoder/16/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  6. _init_1.5B_seed42/0/model/model/decoder/16/pp_block/input_layernorm/model_weight.safetensors +3 -0
  7. _init_1.5B_seed42/0/model/model/decoder/16/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  8. _init_1.5B_seed42/0/model/model/decoder/16/pp_block/post_attention_layernorm/model_weight.safetensors +3 -0
  9. _init_1.5B_seed42/0/model/model/decoder/2/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  10. _init_1.5B_seed42/0/model/model/decoder/2/pp_block/input_layernorm/model_weight.safetensors +3 -0
  11. _init_1.5B_seed42/0/model/model/decoder/2/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  12. _init_1.5B_seed42/0/model/model/decoder/2/pp_block/post_attention_layernorm/model_weight.safetensors +3 -0
  13. _init_1.5B_seed42/0/model/model/decoder/6/pp_block/post_attention_layernorm/model_weight.safetensors +3 -0
  14. _init_1.5B_seed42/0/model/model/final_layer_norm/pp_block/model_weight.safetensors +3 -0
  15. _init_1.5B_seed42/0/model_config.json +1 -0
  16. _init_1.5B_seed42/0/optimizer/optimizer_config.json +1 -0
  17. _init_1.5B_seed42/0/random/tp-0-of-1_dp-0-of-4_pp-0-of-1.pt +3 -0
  18. _init_1.5B_seed42/0/random/tp-0-of-1_dp-1-of-4_pp-0-of-1.pt +3 -0
  19. _init_1.5B_seed42/0/random/tp-0-of-1_dp-2-of-4_pp-0-of-1.pt +3 -0
  20. _init_1.5B_seed42/0/random/tp-0-of-1_dp-3-of-4_pp-0-of-1.pt +3 -0
  21. _init_1.5B_seed43/0/checkpoint_metadata.json +22 -0
  22. _init_1.5B_seed43/0/config.yaml +121 -0
  23. _init_1.5B_seed43/0/model/model/decoder/25/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  24. _init_1.5B_seed43/0/model/model/decoder/25/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  25. _init_1.5B_seed43/0/model/model/decoder/25/pp_block/input_layernorm/model_weight.safetensors +3 -0
  26. _init_1.5B_seed43/0/model/model/decoder/25/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  27. _init_1.5B_seed43/0/model/model/decoder/25/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  28. _init_1.5B_seed43/0/model/model/decoder/25/pp_block/post_attention_layernorm/model_weight.safetensors +3 -0
  29. _init_1.5B_seed43/0/model/model/decoder/4/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  30. _init_1.5B_seed43/0/model/model/decoder/4/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  31. _init_1.5B_seed43/0/model/model/decoder/4/pp_block/input_layernorm/model_weight.safetensors +3 -0
  32. _init_1.5B_seed43/0/model/model/decoder/4/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  33. _init_1.5B_seed43/0/model/model/decoder/4/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  34. _init_1.5B_seed43/0/model/model/decoder/4/pp_block/post_attention_layernorm/model_weight.safetensors +3 -0
  35. _init_1.5B_seed43/0/model/model/decoder/6/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  36. _init_1.5B_seed43/0/model/model/decoder/6/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  37. _init_1.5B_seed43/0/model/model/decoder/6/pp_block/input_layernorm/model_weight.safetensors +3 -0
  38. _init_1.5B_seed43/0/model/model/decoder/6/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  39. _init_1.5B_seed43/0/model/model/decoder/6/pp_block/post_attention_layernorm/model_weight.safetensors +3 -0
  40. _init_1.5B_seed43/0/model/model/final_layer_norm/pp_block/model_weight.safetensors +3 -0
  41. _init_1.5B_seed43/0/model/model/token_position_embeddings/pp_block/token_embedding/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  42. _init_1.5B_seed43/0/model_config.json +1 -0
  43. _init_1.5B_seed43/0/optimizer/optimizer_config.json +1 -0
  44. _init_1.5B_seed44/0/checkpoint_metadata.json +22 -0
  45. _init_1.5B_seed44/0/config.yaml +121 -0
  46. _init_1.5B_seed44/0/model_config.json +1 -0
  47. _init_1.5B_seed44/0/optimizer/optimizer_config.json +1 -0
  48. init_1.5B_seed42.hash.json +179 -0
  49. init_1.5B_seed43.hash.json +179 -0
  50. init_1.5B_seed44.hash.json +179 -0
README.md ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ ---
2
+ pretty_name: Know Your Sources (init checkpoints)
3
+ ---
4
+
5
+ Nanotron sharded checkpoint format. Not loadable with transformers
6
+ `from_pretrained`. Three identical-architecture 1.5B initializations, seeds
7
+ 42 / 43 / 44, used to seed the trunk runs of the 108-config grid.
_init_1.5B_seed42/0/checkpoint_metadata.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cp": 1,
3
+ "custom_metas": null,
4
+ "dp": 4,
5
+ "metas": {
6
+ "consumed_tokens_total": 0,
7
+ "consumed_train_samples": 0,
8
+ "data_stages": [
9
+ {
10
+ "consumed_tokens_per_dataset_folder": {},
11
+ "consumed_train_samples": 0,
12
+ "name": "S0_top10B",
13
+ "sequence_length": 2048,
14
+ "start_training_step": 1
15
+ }
16
+ ],
17
+ "last_stage_idx": 0,
18
+ "last_train_step": 0
19
+ },
20
+ "tp": 1,
21
+ "version": "1.5"
22
+ }
_init_1.5B_seed42/0/config.yaml ADDED
@@ -0,0 +1,121 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ checkpoints:
2
+ checkpoint_interval: 477
3
+ checkpoints_path: /scratch/bvandur1/zhuicon1/checkpoints/quality-base-10B-1.5B-seed42
4
+ checkpoints_path_is_shared_file_system: false
5
+ load_lr_scheduler: true
6
+ load_optimizer: true
7
+ resume_checkpoint_path: null
8
+ save_final_state: true
9
+ save_initial_state: true
10
+ data_stages:
11
+ - data:
12
+ dataset:
13
+ dataset_folder:
14
+ - /scratch/bvandur1/zhuicon1/nanotron_tokenized/10B-base/tokenized
15
+ dataset_max_tokens: null
16
+ dataset_read_path: null
17
+ dataset_weights: null
18
+ pad_samples_to_global_batch_size: false
19
+ return_positions: true
20
+ shuffle_files: false
21
+ skip_in_stream: false
22
+ token_size_in_bytes: 2
23
+ tokenizer_name: /scratch/bvandur1/zhuicon1/tokenizers/llama2-unsloth-tokenizer
24
+ use_old_brrr_dataloader: false
25
+ vocab_size: 32000
26
+ num_loading_workers: 1
27
+ seed: 42
28
+ name: S0_top10B
29
+ sequence_length: 2048
30
+ start_training_step: 1
31
+ general:
32
+ benchmark_csv_path: null
33
+ consumed_train_samples: 0
34
+ ignore_sanity_checks: true
35
+ project: exp-0616
36
+ run: quality-base-10B-1.5B-seed42
37
+ seed: 42
38
+ step: 0
39
+ lighteval: null
40
+ logging:
41
+ iteration_step_info_interval: 1
42
+ log_level: info
43
+ log_level_replica: info
44
+ metrics_logging: null
45
+ model:
46
+ ddp_bucket_cap_mb: 25
47
+ dtype: bfloat16
48
+ init_method:
49
+ scaling_method: NUM_LAYERS
50
+ std: 0.02
51
+ make_vocab_size_divisible_by: 1
52
+ model_config:
53
+ _attn_implementation: flash_attention_2
54
+ attention_bias: false
55
+ bos_token_id: 1
56
+ eos_token_id: 2
57
+ hidden_act: silu
58
+ hidden_size: 2048
59
+ initializer_range: 0.02
60
+ intermediate_size: 5632
61
+ is_llama_config: true
62
+ max_position_embeddings: 2048
63
+ num_attention_heads: 16
64
+ num_hidden_layers: 28
65
+ num_key_value_heads: 16
66
+ pad_token_id: null
67
+ pretraining_tp: 1
68
+ rms_norm_eps: 1.0e-05
69
+ rope_interleaved: false
70
+ rope_scaling: null
71
+ rope_theta: 10000.0
72
+ tie_word_embeddings: true
73
+ use_cache: false
74
+ vocab_size: 32000
75
+ z_loss_coefficient: 0.0001
76
+ z_loss_enabled: false
77
+ optimizer:
78
+ accumulate_grad_in_fp32: true
79
+ clip_grad: 1.0
80
+ learning_rate_scheduler:
81
+ learning_rate: 0.0005
82
+ lr_decay_starting_step: 12875
83
+ lr_decay_steps: 1430
84
+ lr_decay_style: linear
85
+ lr_warmup_steps: 500
86
+ lr_warmup_style: linear
87
+ min_decay_lr: 0.0
88
+ optimizer_factory:
89
+ adam_beta1: 0.9
90
+ adam_beta2: 0.95
91
+ adam_eps: 1.0e-08
92
+ name: adamW
93
+ torch_adam_is_fused: true
94
+ weight_decay: 0.1
95
+ weight_decay_exclude_named_params: []
96
+ zero_stage: 0
97
+ parallelism:
98
+ context_parallel_size: 1
99
+ dp: 4
100
+ expert_parallel_size: 1
101
+ pp: 1
102
+ pp_engine: 1f1b
103
+ recompute_layer: false
104
+ tp: 1
105
+ tp_linear_async_communication: true
106
+ tp_mode: REDUCE_SCATTER
107
+ tp_recompute_allgather: true
108
+ profiler: null
109
+ s3_upload: null
110
+ tokenizer:
111
+ tokenizer_max_length: null
112
+ tokenizer_name_or_path: /scratch/bvandur1/zhuicon1/tokenizers/llama2-unsloth-tokenizer
113
+ tokenizer_revision: null
114
+ tokens:
115
+ batch_accumulation_per_replica: 64
116
+ limit_test_batches: 0
117
+ limit_val_batches: 0
118
+ micro_batch_size: 4
119
+ sequence_length: 2048
120
+ train_steps: 14305
121
+ val_check_interval: -1
_init_1.5B_seed42/0/model/model/decoder/16/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fda8a081e83e52ef3ebe2672d76f0f87a9d86caccd06f8d2eb785617fbeee90f
3
+ size 8388848
_init_1.5B_seed42/0/model/model/decoder/16/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0bd808cd662c3922c0f1d1908f909a3d90c17bfee9ab5e7f8ec4b1dd09e83779
3
+ size 25166192
_init_1.5B_seed42/0/model/model/decoder/16/pp_block/input_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e933f9076c7b4ddb86690e1fe0812223f3a1cbf5305c806a7d6ab40b056f4f7
3
+ size 4192
_init_1.5B_seed42/0/model/model/decoder/16/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6e201d3f55dc3631f4b0193f4d5598f302450fa7e76005e7ae6916755e3fb487
3
+ size 23068912
_init_1.5B_seed42/0/model/model/decoder/16/pp_block/post_attention_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e933f9076c7b4ddb86690e1fe0812223f3a1cbf5305c806a7d6ab40b056f4f7
3
+ size 4192
_init_1.5B_seed42/0/model/model/decoder/2/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6d42a995643475c2d9b62d4e855c8409b3529434457b534f90ac1c191fe6dbe7
3
+ size 8388848
_init_1.5B_seed42/0/model/model/decoder/2/pp_block/input_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e933f9076c7b4ddb86690e1fe0812223f3a1cbf5305c806a7d6ab40b056f4f7
3
+ size 4192
_init_1.5B_seed42/0/model/model/decoder/2/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bd9007de3d95eabd79e78a5733bc65ec3e885073d9e9e21088b42b4672848bc2
3
+ size 23068912
_init_1.5B_seed42/0/model/model/decoder/2/pp_block/post_attention_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e933f9076c7b4ddb86690e1fe0812223f3a1cbf5305c806a7d6ab40b056f4f7
3
+ size 4192
_init_1.5B_seed42/0/model/model/decoder/6/pp_block/post_attention_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e933f9076c7b4ddb86690e1fe0812223f3a1cbf5305c806a7d6ab40b056f4f7
3
+ size 4192
_init_1.5B_seed42/0/model/model/final_layer_norm/pp_block/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e933f9076c7b4ddb86690e1fe0812223f3a1cbf5305c806a7d6ab40b056f4f7
3
+ size 4192
_init_1.5B_seed42/0/model_config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 5632, "is_llama_config": true, "max_position_embeddings": 2048, "num_attention_heads": 16, "num_hidden_layers": 28, "attention_bias": false, "num_key_value_heads": 16, "pad_token_id": null, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000.0, "rope_interleaved": false, "tie_word_embeddings": true, "use_cache": false, "vocab_size": 32000, "_attn_implementation": "flash_attention_2", "z_loss_enabled": false, "z_loss_coefficient": 0.0001}
_init_1.5B_seed42/0/optimizer/optimizer_config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"type": "OptimizerFromGradientAccumulator", "parallelism": {"tp_size": "1", "dp_size": "4", "pp_size": "1", "expert_parallel_size": "1"}, "configs": {}}
_init_1.5B_seed42/0/random/tp-0-of-1_dp-0-of-4_pp-0-of-1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cafa33c7f898af8cfdd14070a86ceb6e82a7e4667f7742eb260461bfa472220c
3
+ size 1080
_init_1.5B_seed42/0/random/tp-0-of-1_dp-1-of-4_pp-0-of-1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9c767793d2f8cbbe122ed6d6d45d61859b9e7c78f012f40becea9d84a766f26d
3
+ size 1080
_init_1.5B_seed42/0/random/tp-0-of-1_dp-2-of-4_pp-0-of-1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fe7792d5682fe9b9672a0f9dff4fd43975f3cb836180000e3eb5a809190a859e
3
+ size 1080
_init_1.5B_seed42/0/random/tp-0-of-1_dp-3-of-4_pp-0-of-1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:456788c887b847f5cba3eb98c3649fe84190ceba1f1ce60dbe40c39ea668703f
3
+ size 1080
_init_1.5B_seed43/0/checkpoint_metadata.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cp": 1,
3
+ "custom_metas": null,
4
+ "dp": 4,
5
+ "metas": {
6
+ "consumed_tokens_total": 0,
7
+ "consumed_train_samples": 0,
8
+ "data_stages": [
9
+ {
10
+ "consumed_tokens_per_dataset_folder": {},
11
+ "consumed_train_samples": 0,
12
+ "name": "S0_init",
13
+ "sequence_length": 2048,
14
+ "start_training_step": 1
15
+ }
16
+ ],
17
+ "last_stage_idx": 0,
18
+ "last_train_step": 0
19
+ },
20
+ "tp": 1,
21
+ "version": "1.5"
22
+ }
_init_1.5B_seed43/0/config.yaml ADDED
@@ -0,0 +1,121 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ checkpoints:
2
+ checkpoint_interval: 1000000
3
+ checkpoints_path: /scratch/bvandur1/zhuicon1/checkpoints/_init_1.5B_seed43
4
+ checkpoints_path_is_shared_file_system: false
5
+ load_lr_scheduler: true
6
+ load_optimizer: true
7
+ resume_checkpoint_path: null
8
+ save_final_state: false
9
+ save_initial_state: true
10
+ data_stages:
11
+ - data:
12
+ dataset:
13
+ dataset_folder:
14
+ - /scratch/bvandur1/zhuicon1/nanotron_tokenized_7B/quality-base/tokenized
15
+ dataset_max_tokens: null
16
+ dataset_read_path: null
17
+ dataset_weights: null
18
+ pad_samples_to_global_batch_size: false
19
+ return_positions: true
20
+ shuffle_files: false
21
+ skip_in_stream: false
22
+ token_size_in_bytes: 2
23
+ tokenizer_name: /scratch/bvandur1/zhuicon1/tokenizers/llama2-unsloth-tokenizer
24
+ use_old_brrr_dataloader: false
25
+ vocab_size: 32000
26
+ num_loading_workers: 1
27
+ seed: 43
28
+ name: S0_init
29
+ sequence_length: 2048
30
+ start_training_step: 1
31
+ general:
32
+ benchmark_csv_path: null
33
+ consumed_train_samples: 0
34
+ ignore_sanity_checks: true
35
+ project: exp-0626
36
+ run: _init_1.5B_seed43_gen
37
+ seed: 43
38
+ step: 0
39
+ lighteval: null
40
+ logging:
41
+ iteration_step_info_interval: 1
42
+ log_level: info
43
+ log_level_replica: info
44
+ metrics_logging: null
45
+ model:
46
+ ddp_bucket_cap_mb: 25
47
+ dtype: bfloat16
48
+ init_method:
49
+ scaling_method: NUM_LAYERS
50
+ std: 0.02
51
+ make_vocab_size_divisible_by: 1
52
+ model_config:
53
+ _attn_implementation: flash_attention_2
54
+ attention_bias: false
55
+ bos_token_id: 1
56
+ eos_token_id: 2
57
+ hidden_act: silu
58
+ hidden_size: 2048
59
+ initializer_range: 0.02
60
+ intermediate_size: 5632
61
+ is_llama_config: true
62
+ max_position_embeddings: 2048
63
+ num_attention_heads: 16
64
+ num_hidden_layers: 28
65
+ num_key_value_heads: 16
66
+ pad_token_id: null
67
+ pretraining_tp: 1
68
+ rms_norm_eps: 1.0e-05
69
+ rope_interleaved: false
70
+ rope_scaling: null
71
+ rope_theta: 10000.0
72
+ tie_word_embeddings: true
73
+ use_cache: false
74
+ vocab_size: 32000
75
+ z_loss_coefficient: 0.0001
76
+ z_loss_enabled: false
77
+ optimizer:
78
+ accumulate_grad_in_fp32: true
79
+ clip_grad: 1.0
80
+ learning_rate_scheduler:
81
+ learning_rate: 0.0005
82
+ lr_decay_starting_step: 1
83
+ lr_decay_steps: 0
84
+ lr_decay_style: linear
85
+ lr_warmup_steps: 0
86
+ lr_warmup_style: linear
87
+ min_decay_lr: 0.0
88
+ optimizer_factory:
89
+ adam_beta1: 0.9
90
+ adam_beta2: 0.95
91
+ adam_eps: 1.0e-08
92
+ name: adamW
93
+ torch_adam_is_fused: true
94
+ weight_decay: 0.1
95
+ weight_decay_exclude_named_params: []
96
+ zero_stage: 0
97
+ parallelism:
98
+ context_parallel_size: 1
99
+ dp: 4
100
+ expert_parallel_size: 1
101
+ pp: 1
102
+ pp_engine: 1f1b
103
+ recompute_layer: false
104
+ tp: 1
105
+ tp_linear_async_communication: true
106
+ tp_mode: REDUCE_SCATTER
107
+ tp_recompute_allgather: true
108
+ profiler: null
109
+ s3_upload: null
110
+ tokenizer:
111
+ tokenizer_max_length: null
112
+ tokenizer_name_or_path: /scratch/bvandur1/zhuicon1/tokenizers/llama2-unsloth-tokenizer
113
+ tokenizer_revision: null
114
+ tokens:
115
+ batch_accumulation_per_replica: 64
116
+ limit_test_batches: 0
117
+ limit_val_batches: 0
118
+ micro_batch_size: 4
119
+ sequence_length: 2048
120
+ train_steps: 1
121
+ val_check_interval: -1
_init_1.5B_seed43/0/model/model/decoder/25/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:263d3aca6a30b932c25c07add5dabd7bd6efe137c4003aa6fa62ff159e8efba7
3
+ size 8388848
_init_1.5B_seed43/0/model/model/decoder/25/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d55ca32577e83d73ec4b1d8c0256155415eccb59325c2ca203c63237fce86c84
3
+ size 25166192
_init_1.5B_seed43/0/model/model/decoder/25/pp_block/input_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e933f9076c7b4ddb86690e1fe0812223f3a1cbf5305c806a7d6ab40b056f4f7
3
+ size 4192
_init_1.5B_seed43/0/model/model/decoder/25/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:535c82edf4fa5ecdf7edfb1c2577b38da2d756b63a7e552bf236944f1dce1819
3
+ size 23068912
_init_1.5B_seed43/0/model/model/decoder/25/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ea7891931a3797cd9804a47ab5338f52f50e5f54f8741877b0b8d17d5fcb0821
3
+ size 46137656
_init_1.5B_seed43/0/model/model/decoder/25/pp_block/post_attention_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e933f9076c7b4ddb86690e1fe0812223f3a1cbf5305c806a7d6ab40b056f4f7
3
+ size 4192
_init_1.5B_seed43/0/model/model/decoder/4/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:87990597239494a501fcfc53388c1ab9020dde179e04e8c5d3a9f26694f09b1b
3
+ size 8388848
_init_1.5B_seed43/0/model/model/decoder/4/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ef3389eae1075d5571b9bd09e0c7cff7c0e47605bc61f872e948895249733e28
3
+ size 25166192
_init_1.5B_seed43/0/model/model/decoder/4/pp_block/input_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e933f9076c7b4ddb86690e1fe0812223f3a1cbf5305c806a7d6ab40b056f4f7
3
+ size 4192
_init_1.5B_seed43/0/model/model/decoder/4/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7b27a4cbe0346b75964e1fc5795cff01b20b7cbd8ab885a3fc6698a551e2de14
3
+ size 23068912
_init_1.5B_seed43/0/model/model/decoder/4/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3449629ed19dce8fdcf07f0a29b4ee6ee32f2e14b931cb4afa4ff13a0329d1a7
3
+ size 46137656
_init_1.5B_seed43/0/model/model/decoder/4/pp_block/post_attention_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e933f9076c7b4ddb86690e1fe0812223f3a1cbf5305c806a7d6ab40b056f4f7
3
+ size 4192
_init_1.5B_seed43/0/model/model/decoder/6/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6122fc58a3ff83068fc10d9709f41b7cc6731c3854fbb621eb3a1bb88a82b2f7
3
+ size 8388848
_init_1.5B_seed43/0/model/model/decoder/6/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a6ba1fb8639cd8a3eebe28e7ba610a22fcaae207e313c8e99e1b273ca9fd4600
3
+ size 25166192
_init_1.5B_seed43/0/model/model/decoder/6/pp_block/input_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e933f9076c7b4ddb86690e1fe0812223f3a1cbf5305c806a7d6ab40b056f4f7
3
+ size 4192
_init_1.5B_seed43/0/model/model/decoder/6/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:77d848007d4dad255694c1e66ad5cfe46e250b35f24f7caf393b757303aac3a1
3
+ size 23068912
_init_1.5B_seed43/0/model/model/decoder/6/pp_block/post_attention_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e933f9076c7b4ddb86690e1fe0812223f3a1cbf5305c806a7d6ab40b056f4f7
3
+ size 4192
_init_1.5B_seed43/0/model/model/final_layer_norm/pp_block/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e933f9076c7b4ddb86690e1fe0812223f3a1cbf5305c806a7d6ab40b056f4f7
3
+ size 4192
_init_1.5B_seed43/0/model/model/token_position_embeddings/pp_block/token_embedding/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6cea3f31d69f56a70232db58a7b65c392883cb4fb7474bfc3adf902b6f131734
3
+ size 131072240
_init_1.5B_seed43/0/model_config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 5632, "is_llama_config": true, "max_position_embeddings": 2048, "num_attention_heads": 16, "num_hidden_layers": 28, "attention_bias": false, "num_key_value_heads": 16, "pad_token_id": null, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000.0, "rope_interleaved": false, "tie_word_embeddings": true, "use_cache": false, "vocab_size": 32000, "_attn_implementation": "flash_attention_2", "z_loss_enabled": false, "z_loss_coefficient": 0.0001}
_init_1.5B_seed43/0/optimizer/optimizer_config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"type": "OptimizerFromGradientAccumulator", "parallelism": {"tp_size": "1", "dp_size": "4", "pp_size": "1", "expert_parallel_size": "1"}, "configs": {}}
_init_1.5B_seed44/0/checkpoint_metadata.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cp": 1,
3
+ "custom_metas": null,
4
+ "dp": 4,
5
+ "metas": {
6
+ "consumed_tokens_total": 0,
7
+ "consumed_train_samples": 0,
8
+ "data_stages": [
9
+ {
10
+ "consumed_tokens_per_dataset_folder": {},
11
+ "consumed_train_samples": 0,
12
+ "name": "S0_init",
13
+ "sequence_length": 2048,
14
+ "start_training_step": 1
15
+ }
16
+ ],
17
+ "last_stage_idx": 0,
18
+ "last_train_step": 0
19
+ },
20
+ "tp": 1,
21
+ "version": "1.5"
22
+ }
_init_1.5B_seed44/0/config.yaml ADDED
@@ -0,0 +1,121 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ checkpoints:
2
+ checkpoint_interval: 1000000
3
+ checkpoints_path: /scratch/bvandur1/zhuicon1/checkpoints/_init_1.5B_seed44
4
+ checkpoints_path_is_shared_file_system: false
5
+ load_lr_scheduler: true
6
+ load_optimizer: true
7
+ resume_checkpoint_path: null
8
+ save_final_state: false
9
+ save_initial_state: true
10
+ data_stages:
11
+ - data:
12
+ dataset:
13
+ dataset_folder:
14
+ - /scratch/bvandur1/zhuicon1/nanotron_tokenized_7B/quality-base/tokenized
15
+ dataset_max_tokens: null
16
+ dataset_read_path: null
17
+ dataset_weights: null
18
+ pad_samples_to_global_batch_size: false
19
+ return_positions: true
20
+ shuffle_files: false
21
+ skip_in_stream: false
22
+ token_size_in_bytes: 2
23
+ tokenizer_name: /scratch/bvandur1/zhuicon1/tokenizers/llama2-unsloth-tokenizer
24
+ use_old_brrr_dataloader: false
25
+ vocab_size: 32000
26
+ num_loading_workers: 1
27
+ seed: 44
28
+ name: S0_init
29
+ sequence_length: 2048
30
+ start_training_step: 1
31
+ general:
32
+ benchmark_csv_path: null
33
+ consumed_train_samples: 0
34
+ ignore_sanity_checks: true
35
+ project: exp-0626
36
+ run: _init_1.5B_seed44_gen
37
+ seed: 44
38
+ step: 0
39
+ lighteval: null
40
+ logging:
41
+ iteration_step_info_interval: 1
42
+ log_level: info
43
+ log_level_replica: info
44
+ metrics_logging: null
45
+ model:
46
+ ddp_bucket_cap_mb: 25
47
+ dtype: bfloat16
48
+ init_method:
49
+ scaling_method: NUM_LAYERS
50
+ std: 0.02
51
+ make_vocab_size_divisible_by: 1
52
+ model_config:
53
+ _attn_implementation: flash_attention_2
54
+ attention_bias: false
55
+ bos_token_id: 1
56
+ eos_token_id: 2
57
+ hidden_act: silu
58
+ hidden_size: 2048
59
+ initializer_range: 0.02
60
+ intermediate_size: 5632
61
+ is_llama_config: true
62
+ max_position_embeddings: 2048
63
+ num_attention_heads: 16
64
+ num_hidden_layers: 28
65
+ num_key_value_heads: 16
66
+ pad_token_id: null
67
+ pretraining_tp: 1
68
+ rms_norm_eps: 1.0e-05
69
+ rope_interleaved: false
70
+ rope_scaling: null
71
+ rope_theta: 10000.0
72
+ tie_word_embeddings: true
73
+ use_cache: false
74
+ vocab_size: 32000
75
+ z_loss_coefficient: 0.0001
76
+ z_loss_enabled: false
77
+ optimizer:
78
+ accumulate_grad_in_fp32: true
79
+ clip_grad: 1.0
80
+ learning_rate_scheduler:
81
+ learning_rate: 0.0005
82
+ lr_decay_starting_step: 1
83
+ lr_decay_steps: 0
84
+ lr_decay_style: linear
85
+ lr_warmup_steps: 0
86
+ lr_warmup_style: linear
87
+ min_decay_lr: 0.0
88
+ optimizer_factory:
89
+ adam_beta1: 0.9
90
+ adam_beta2: 0.95
91
+ adam_eps: 1.0e-08
92
+ name: adamW
93
+ torch_adam_is_fused: true
94
+ weight_decay: 0.1
95
+ weight_decay_exclude_named_params: []
96
+ zero_stage: 0
97
+ parallelism:
98
+ context_parallel_size: 1
99
+ dp: 4
100
+ expert_parallel_size: 1
101
+ pp: 1
102
+ pp_engine: 1f1b
103
+ recompute_layer: false
104
+ tp: 1
105
+ tp_linear_async_communication: true
106
+ tp_mode: REDUCE_SCATTER
107
+ tp_recompute_allgather: true
108
+ profiler: null
109
+ s3_upload: null
110
+ tokenizer:
111
+ tokenizer_max_length: null
112
+ tokenizer_name_or_path: /scratch/bvandur1/zhuicon1/tokenizers/llama2-unsloth-tokenizer
113
+ tokenizer_revision: null
114
+ tokens:
115
+ batch_accumulation_per_replica: 64
116
+ limit_test_batches: 0
117
+ limit_val_batches: 0
118
+ micro_batch_size: 4
119
+ sequence_length: 2048
120
+ train_steps: 1
121
+ val_check_interval: -1
_init_1.5B_seed44/0/model_config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"bos_token_id": 1, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 2048, "initializer_range": 0.02, "intermediate_size": 5632, "is_llama_config": true, "max_position_embeddings": 2048, "num_attention_heads": 16, "num_hidden_layers": 28, "attention_bias": false, "num_key_value_heads": 16, "pad_token_id": null, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": null, "rope_theta": 10000.0, "rope_interleaved": false, "tie_word_embeddings": true, "use_cache": false, "vocab_size": 32000, "_attn_implementation": "flash_attention_2", "z_loss_enabled": false, "z_loss_coefficient": 0.0001}
_init_1.5B_seed44/0/optimizer/optimizer_config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"type": "OptimizerFromGradientAccumulator", "parallelism": {"tp_size": "1", "dp_size": "4", "pp_size": "1", "expert_parallel_size": "1"}, "configs": {}}
init_1.5B_seed42.hash.json ADDED
@@ -0,0 +1,179 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": "/scratch/bvandur1/zhuicon1/checkpoints/_init_1.5B_seed42/0",
3
+ "n_tensor_files": 170,
4
+ "n_tensors": 170,
5
+ "n_parameters": 1504299008,
6
+ "rolling_sha256": "2ede6612b2e48d7529f867f0e74ca0a7d9ba79cd635d4f803f8022d9b0113aba",
7
+ "per_tensor": {
8
+ "model/decoder/0/pp_block/attn/o_proj/model_weight::data": "f1af5b48711468e05152fa51054c6777b1314f4123422396714719f9cb01f34c",
9
+ "model/decoder/0/pp_block/attn/qkv_proj/model_weight::data": "64a4edc5a80a6531a3273cd869a8c81f33b1a2d0e2bdb43c9614605ad38d0add",
10
+ "model/decoder/0/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
11
+ "model/decoder/0/pp_block/mlp/down_proj/model_weight::data": "4688c4277b7b49f2840adce2e26f7c560dd9f1508d7c1da61142b7248a4189f4",
12
+ "model/decoder/0/pp_block/mlp/gate_up_proj/model_weight::data": "9ffa779de658d1912d31fc21d71fe910eadc7cf3762e14c3e59e0c37653f0bc3",
13
+ "model/decoder/0/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
14
+ "model/decoder/1/pp_block/attn/o_proj/model_weight::data": "a1322205a1ffffb7a420b1884fd7cd857c1ec12d94513621d7cbb4c5d34a3618",
15
+ "model/decoder/1/pp_block/attn/qkv_proj/model_weight::data": "0f91d458410c4709ec4aca71726ff032d356160817ff7b76becb4e5e1c6463ab",
16
+ "model/decoder/1/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
17
+ "model/decoder/1/pp_block/mlp/down_proj/model_weight::data": "500c2c5fb7925f686f4a96954071299b52dc61f52c8a6f9a84234dca74a46a75",
18
+ "model/decoder/1/pp_block/mlp/gate_up_proj/model_weight::data": "8c1b4b1060664c4fd8e885865d15706c4a2df670f47c51984c16f58f2a2fbae5",
19
+ "model/decoder/1/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
20
+ "model/decoder/10/pp_block/attn/o_proj/model_weight::data": "2b33d33f7ab4dbf06265e541cfc6cf824b2de19337bcb072c97f4c12ad56f735",
21
+ "model/decoder/10/pp_block/attn/qkv_proj/model_weight::data": "a1ef003721041eca65d50f7a80e6f0924bc2c3d32bfc148d3a93a66aaa007b53",
22
+ "model/decoder/10/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
23
+ "model/decoder/10/pp_block/mlp/down_proj/model_weight::data": "00dcfcbba040de7616be096261c22b6b290aaeeb1eabad08ccdb92df8daf461a",
24
+ "model/decoder/10/pp_block/mlp/gate_up_proj/model_weight::data": "ec99812cc46d36e53e7d846cffbd03648b10a7434b26b7325848851e6864f514",
25
+ "model/decoder/10/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
26
+ "model/decoder/11/pp_block/attn/o_proj/model_weight::data": "0947e580c7abce27e0504e4a5a41b3223d46df2beabb9f088d1c15106b523389",
27
+ "model/decoder/11/pp_block/attn/qkv_proj/model_weight::data": "bbb4ea655ec6c5f7465464ae763cc8b9c10f5af3d077d70f594362f5e3f12a9f",
28
+ "model/decoder/11/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
29
+ "model/decoder/11/pp_block/mlp/down_proj/model_weight::data": "e02a52142e6dee617661a9ff9afa1114a1c2e0db5e31676a443a6404dd7bbe3a",
30
+ "model/decoder/11/pp_block/mlp/gate_up_proj/model_weight::data": "7009b1628645be41fa14cc949f82a4e75ab5b945db5952317bcc78cbb2fd2531",
31
+ "model/decoder/11/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
32
+ "model/decoder/12/pp_block/attn/o_proj/model_weight::data": "17216779661f032485ffe48f89483e8cf2730f37eca0bc75414938a3057627c6",
33
+ "model/decoder/12/pp_block/attn/qkv_proj/model_weight::data": "40067495b805356cc997d95fad6a9c677f22b75143b3bd93ff983636edec5e9f",
34
+ "model/decoder/12/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
35
+ "model/decoder/12/pp_block/mlp/down_proj/model_weight::data": "05ac37d6c1ac46393df1060b5aad0f949ea772cfd1079a173fbbc472154a9f7e",
36
+ "model/decoder/12/pp_block/mlp/gate_up_proj/model_weight::data": "a1091abc0703c87fcd2e8050cc93c0ee52e031746bbf8f9da9b0d564c47eb8ee",
37
+ "model/decoder/12/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
38
+ "model/decoder/13/pp_block/attn/o_proj/model_weight::data": "e85103c6afea97557dc2e7dd948326541fd3aa0f266f28f5118e51e01884384c",
39
+ "model/decoder/13/pp_block/attn/qkv_proj/model_weight::data": "29c49a4cc84d570ef732d63f1639f17adea6cdc5d3efb4384b79421070be5334",
40
+ "model/decoder/13/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
41
+ "model/decoder/13/pp_block/mlp/down_proj/model_weight::data": "46c997a64ab542e77c06992d48ad577baad40ff1af57042adf23bed6b208a1c8",
42
+ "model/decoder/13/pp_block/mlp/gate_up_proj/model_weight::data": "eb975a8045ef5555a3b68eb0b2867db82cb588736011be6fcd2e60f59ec5e548",
43
+ "model/decoder/13/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
44
+ "model/decoder/14/pp_block/attn/o_proj/model_weight::data": "45156234b1d3701d2b5c1a50d619f38f68864e0929d4d9be8ca866c464944e2c",
45
+ "model/decoder/14/pp_block/attn/qkv_proj/model_weight::data": "bc1c448c93e11a92e6260933b212bbcfbe45656de82cf1ebe93e6c8215ab2fe9",
46
+ "model/decoder/14/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
47
+ "model/decoder/14/pp_block/mlp/down_proj/model_weight::data": "bfa5bfa85c6b7f8ce04f7fae7dcf46ae98d580e2ea0092ecf3b62864bbbb01ce",
48
+ "model/decoder/14/pp_block/mlp/gate_up_proj/model_weight::data": "e9b79b0de4773c1dbeba80adc69064245f1978e72b26a1caf2f84c773c2499fd",
49
+ "model/decoder/14/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
50
+ "model/decoder/15/pp_block/attn/o_proj/model_weight::data": "6c92993acf1ae273a202d6c2015d7d387f4ca4e647100424aed8a2816ece6f2f",
51
+ "model/decoder/15/pp_block/attn/qkv_proj/model_weight::data": "8ecb4a8a489e70870f88a5bd4287861409c972c0d94c7275cfdad18ae3086593",
52
+ "model/decoder/15/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
53
+ "model/decoder/15/pp_block/mlp/down_proj/model_weight::data": "f17eee798ad2ff72ff9c0575a261c3201515d24861363c1fd4e761bf383e72f5",
54
+ "model/decoder/15/pp_block/mlp/gate_up_proj/model_weight::data": "551fb20744074c79df11d82c9ecd005b7952a085fd5dedf3652b2c8f240ea3e0",
55
+ "model/decoder/15/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
56
+ "model/decoder/16/pp_block/attn/o_proj/model_weight::data": "d5dbf9171917005d7bfbf90609c4386280b7151127325a3a9f0ccb997b49f16c",
57
+ "model/decoder/16/pp_block/attn/qkv_proj/model_weight::data": "c09dfd4f66886f6268b06e4000f9e8fa642f9b9fbeb4ec9f71360a3f9a0b1015",
58
+ "model/decoder/16/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
59
+ "model/decoder/16/pp_block/mlp/down_proj/model_weight::data": "42f91cf64ae9ee1f124a8c6cc0b93a2b9f4fbff0288c164b8e18cbf2c990a51e",
60
+ "model/decoder/16/pp_block/mlp/gate_up_proj/model_weight::data": "f3e258081ed3603413d771671d9690517911b50ffbba8b3afd3382e242c4ba1c",
61
+ "model/decoder/16/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
62
+ "model/decoder/17/pp_block/attn/o_proj/model_weight::data": "42f4290d8c01dc73fcdad8d02284f61b8bba9e9bd2a8d8e0a705d7ff9fbeefbc",
63
+ "model/decoder/17/pp_block/attn/qkv_proj/model_weight::data": "89e511f150abd73a13fa6e5b6dbe266f2223ee34187af1b16532c3b592b479a0",
64
+ "model/decoder/17/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
65
+ "model/decoder/17/pp_block/mlp/down_proj/model_weight::data": "fc480961721644122dca75f262976721b455bad92537bf6a9a7e93cf2d282116",
66
+ "model/decoder/17/pp_block/mlp/gate_up_proj/model_weight::data": "29280764abcf848eb4a2e645d641908eaeed6635d31b8e038e2896e30e4785a5",
67
+ "model/decoder/17/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
68
+ "model/decoder/18/pp_block/attn/o_proj/model_weight::data": "db89a43f2ebd476acda3a90ee33765add77d2e6dae73fe0ba5697181f70bae0f",
69
+ "model/decoder/18/pp_block/attn/qkv_proj/model_weight::data": "88e76f5503526611b246b9b5ac26364c995ff2c5063f440e4ccc51b32d791d1b",
70
+ "model/decoder/18/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
71
+ "model/decoder/18/pp_block/mlp/down_proj/model_weight::data": "0719282eba7c9a92747b15ac29ac2c00ccb39b9b2e70ba3d712f1c93205cbb17",
72
+ "model/decoder/18/pp_block/mlp/gate_up_proj/model_weight::data": "4ba9191229708316e79b39f168d44fdb7d8d850823baac36b5434d1be131c747",
73
+ "model/decoder/18/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
74
+ "model/decoder/19/pp_block/attn/o_proj/model_weight::data": "5cc844c0fbf2b827ec537eed27a8eaa6787d16b8d25b21a0184f9cf993a2b920",
75
+ "model/decoder/19/pp_block/attn/qkv_proj/model_weight::data": "563d07deb988e5f9d8f0776e83b939ccbb2f02e487d6352953925b0eacc139e9",
76
+ "model/decoder/19/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
77
+ "model/decoder/19/pp_block/mlp/down_proj/model_weight::data": "98a4ba80c9f78897e1376e8ec1c1b223b781281eeee9da74d39edc3f4995b043",
78
+ "model/decoder/19/pp_block/mlp/gate_up_proj/model_weight::data": "9bca8a2c2967ec3273d6140d402504cdbc803f21e8837d934e88f3277fbbe64d",
79
+ "model/decoder/19/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
80
+ "model/decoder/2/pp_block/attn/o_proj/model_weight::data": "2de3021ae5715ebf60c628b78ebb3d67583d666dd6521918427354ba6e49aeb9",
81
+ "model/decoder/2/pp_block/attn/qkv_proj/model_weight::data": "8ff3fd6c25081053016fbb29da914cb94394d2effc1072d2e496b1ce04bc28ee",
82
+ "model/decoder/2/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
83
+ "model/decoder/2/pp_block/mlp/down_proj/model_weight::data": "bf33a22942d2bfb920921cfd670d2410657727edd4f82281d3368c404ab43e2e",
84
+ "model/decoder/2/pp_block/mlp/gate_up_proj/model_weight::data": "bfa0f726afd4bb0212c6614db225a1797d29fff27933626bfe79dc0bd5a26e93",
85
+ "model/decoder/2/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
86
+ "model/decoder/20/pp_block/attn/o_proj/model_weight::data": "5a0fac6baecae5e2eae7b31669f56e35e18abd08aafcaf2066450744c69aebf9",
87
+ "model/decoder/20/pp_block/attn/qkv_proj/model_weight::data": "05afce8c93e4c01bff95da6bd9f45fabdbc0ca8c759e6a03c7d282fb27ada77d",
88
+ "model/decoder/20/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
89
+ "model/decoder/20/pp_block/mlp/down_proj/model_weight::data": "1adf58f2c006904d57e66a2d8745543ac115a23d51207c8f196d2162d3539b75",
90
+ "model/decoder/20/pp_block/mlp/gate_up_proj/model_weight::data": "99335d7f3dcea29073df430cc820615df0c60cfc40d76e78766272044c2d670d",
91
+ "model/decoder/20/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
92
+ "model/decoder/21/pp_block/attn/o_proj/model_weight::data": "857bffe8033b90347adbf4a66e93dfc8a010ae275b4fd0ea6a3e7e9800a7a237",
93
+ "model/decoder/21/pp_block/attn/qkv_proj/model_weight::data": "9c5426d2ea87ad37d1d4b8948a48243f76a91dbaae668e052ec75b56e4b6ee20",
94
+ "model/decoder/21/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
95
+ "model/decoder/21/pp_block/mlp/down_proj/model_weight::data": "0a93f84487a438aa09f5c6d88ed8743814e2f5ad5161253858bcffbc7efcfc03",
96
+ "model/decoder/21/pp_block/mlp/gate_up_proj/model_weight::data": "71f6b692fad82508d52ab6146fe1f8b341571db95831d98838f26549b8e2096b",
97
+ "model/decoder/21/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
98
+ "model/decoder/22/pp_block/attn/o_proj/model_weight::data": "8825b3af75a1fc392862f5aff2648c8ada6216316875bda3c6fa1a7c68974554",
99
+ "model/decoder/22/pp_block/attn/qkv_proj/model_weight::data": "c5155284ef6170927fbc2d4f8f7b535025fbfa86b57773c17e26034c115f2334",
100
+ "model/decoder/22/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
101
+ "model/decoder/22/pp_block/mlp/down_proj/model_weight::data": "22df3d164c90b0c3a6c37cad19c610815a894ac128746e0e8027b9ea9e45e70a",
102
+ "model/decoder/22/pp_block/mlp/gate_up_proj/model_weight::data": "2565142c545b07dd119b870b602caeaa4fe47efacf4c41c5a6deca671e0d4f98",
103
+ "model/decoder/22/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
104
+ "model/decoder/23/pp_block/attn/o_proj/model_weight::data": "c9abf55596aeee01d3d44d990ce8219d28c0bbfda76546c957087ea8937982c9",
105
+ "model/decoder/23/pp_block/attn/qkv_proj/model_weight::data": "b91b7705d942be1d088da27cdae34e32c0709b4f5acb661803534a7584c9888b",
106
+ "model/decoder/23/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
107
+ "model/decoder/23/pp_block/mlp/down_proj/model_weight::data": "9cef3cb98d30e21d0385758307d6b0f950bf751e956c3257090defa1e9925a98",
108
+ "model/decoder/23/pp_block/mlp/gate_up_proj/model_weight::data": "1dd7d1b98963fcfd8c68f83940bd5335524ffcc1a8a8297f5719c921a3138b63",
109
+ "model/decoder/23/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
110
+ "model/decoder/24/pp_block/attn/o_proj/model_weight::data": "71141f7799a02b0fbff38dc599880ad6a0d94004c3a4b0cd32758bbf98fba454",
111
+ "model/decoder/24/pp_block/attn/qkv_proj/model_weight::data": "47dfb6b388d08572b93cb84115c01539cf59b160f7e5c909bb218cd5bdce3701",
112
+ "model/decoder/24/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
113
+ "model/decoder/24/pp_block/mlp/down_proj/model_weight::data": "d0df3f05dd6b97d2e5d4e8a15b6aa3bf510cf27e0e2edf63bade76b0db655883",
114
+ "model/decoder/24/pp_block/mlp/gate_up_proj/model_weight::data": "e95d820167157fb95ff8d53d13062aef8899792dff157def7830a25135558a01",
115
+ "model/decoder/24/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
116
+ "model/decoder/25/pp_block/attn/o_proj/model_weight::data": "b2b8554ed5a7ea686dda78bb7103cdb8584e518a4ee45a98e620baf8cadcecec",
117
+ "model/decoder/25/pp_block/attn/qkv_proj/model_weight::data": "36c197cea1de9da2955a9d5f2dedc3624d70ef7ce7de63be3390f9690bf790f0",
118
+ "model/decoder/25/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
119
+ "model/decoder/25/pp_block/mlp/down_proj/model_weight::data": "25faf56e6dd6281283f3fcc58c6ed682c313c075d3420cb60231007f98a4414c",
120
+ "model/decoder/25/pp_block/mlp/gate_up_proj/model_weight::data": "8a8c7d6483ac3e0cca82e81d82e0b02a8ef17bc6ec7b0fba0e0a1ed71a3c83b8",
121
+ "model/decoder/25/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
122
+ "model/decoder/26/pp_block/attn/o_proj/model_weight::data": "86e475e5f60c6e941d7cf89797fd861d343005b2e7a2f4c952ca8eb1899ab7e6",
123
+ "model/decoder/26/pp_block/attn/qkv_proj/model_weight::data": "c5a8e4ad59b4accd52e1d1825f2d62eb2a89ce48232414b08afba3ee21132f75",
124
+ "model/decoder/26/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
125
+ "model/decoder/26/pp_block/mlp/down_proj/model_weight::data": "98d78b5b7d1ff66492fa37ef1011c303be2cb2292f61dd3b78beda4625a42c8b",
126
+ "model/decoder/26/pp_block/mlp/gate_up_proj/model_weight::data": "47bd8678128e1f433ae22f3e725ee3777eb5f15fc70a9879f3ec9abc8c1aeb20",
127
+ "model/decoder/26/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
128
+ "model/decoder/27/pp_block/attn/o_proj/model_weight::data": "a55d92dd51c0072dc3554098fb2f703c76122dd26eaf642e231e84be2f3985d0",
129
+ "model/decoder/27/pp_block/attn/qkv_proj/model_weight::data": "2d02dbf64826cf81b2f56e818396e6397993a338af61468078d9f69373f5eb5c",
130
+ "model/decoder/27/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
131
+ "model/decoder/27/pp_block/mlp/down_proj/model_weight::data": "d71e1fe83266b83e658dbd91f4df9ef9a4b9054a9b0b54681df787c4a805ee57",
132
+ "model/decoder/27/pp_block/mlp/gate_up_proj/model_weight::data": "71f9c61301bdef947c774a82bd699336d7111b2d893694cca6ea94bfcd463cd0",
133
+ "model/decoder/27/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
134
+ "model/decoder/3/pp_block/attn/o_proj/model_weight::data": "bf2ea0b3c78a85ac4c864e6bc79b5b640f7f1e82f7bcf2ec23ee6a1ee644fb2c",
135
+ "model/decoder/3/pp_block/attn/qkv_proj/model_weight::data": "14a88ef04a2a3fb3adb39972cf3746c6e52a164e20f760915529819896016564",
136
+ "model/decoder/3/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
137
+ "model/decoder/3/pp_block/mlp/down_proj/model_weight::data": "90dc17dcb9c2d9f3203b65dab22602dac62b9f56ae585c3b7005c5792a894d1a",
138
+ "model/decoder/3/pp_block/mlp/gate_up_proj/model_weight::data": "d1bc0950d7d860ff1b02e751d8f254a7b400bc003dd94287ca7dfc170c1ec6d0",
139
+ "model/decoder/3/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
140
+ "model/decoder/4/pp_block/attn/o_proj/model_weight::data": "b65b79e87c6fcef20c1c55c282842c16fadb10d3ec737210a65442c66dd883f1",
141
+ "model/decoder/4/pp_block/attn/qkv_proj/model_weight::data": "9e923f328d7a09a2dc27c595dd27425eb2979f7d0dfa5a2c51bfe336a69e15ea",
142
+ "model/decoder/4/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
143
+ "model/decoder/4/pp_block/mlp/down_proj/model_weight::data": "186f0d477e08c57a7e6a482667a2838848a70cd6d293ba7da7eb65194b16a6aa",
144
+ "model/decoder/4/pp_block/mlp/gate_up_proj/model_weight::data": "163b123d8c5dc96c5068203a2457677e3e5f76abfe283c4eadc2268d94d0c345",
145
+ "model/decoder/4/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
146
+ "model/decoder/5/pp_block/attn/o_proj/model_weight::data": "2af8e004a1537ab6094d1380068eb37bd8a1919004ca16baab3c7cc7c3b9236c",
147
+ "model/decoder/5/pp_block/attn/qkv_proj/model_weight::data": "973a9524bd0f8ed7727c9b7126d876436179644b69544a2abb7471daa2fff757",
148
+ "model/decoder/5/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
149
+ "model/decoder/5/pp_block/mlp/down_proj/model_weight::data": "1acc335b78fd120873b1084bcb586a664af70af5c8ed5b1087a2f4692fa559df",
150
+ "model/decoder/5/pp_block/mlp/gate_up_proj/model_weight::data": "7e801ae019caa4d9eb559a2d65a05523a3111dc4e5c7631f96014ba92e9747e2",
151
+ "model/decoder/5/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
152
+ "model/decoder/6/pp_block/attn/o_proj/model_weight::data": "f95d609e37eb5f0b68f35cf2d4deb14490d676c72a2fad055cf03f41b5de32b8",
153
+ "model/decoder/6/pp_block/attn/qkv_proj/model_weight::data": "981c621957d64de3b10b691d47e99f20fd61668c87c050504c4ffde3fc43a2db",
154
+ "model/decoder/6/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
155
+ "model/decoder/6/pp_block/mlp/down_proj/model_weight::data": "1b46fe339278fb0bcb25a9bb69699f0481a470e77e0a7af06696868cddcdbbe3",
156
+ "model/decoder/6/pp_block/mlp/gate_up_proj/model_weight::data": "cfa0a37105d830671b807513780d0421c43fdc1094fe20208152923e825a47ea",
157
+ "model/decoder/6/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
158
+ "model/decoder/7/pp_block/attn/o_proj/model_weight::data": "e1f83ab96934e1d6dc0c86e66ecc700937c0e8a374e1cd3f0802d10b371b7c03",
159
+ "model/decoder/7/pp_block/attn/qkv_proj/model_weight::data": "c7ccbc0bcbb9845e8b5cc97bfd9ac43f7ab35aa0449f978731f0946c7811e499",
160
+ "model/decoder/7/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
161
+ "model/decoder/7/pp_block/mlp/down_proj/model_weight::data": "96d604c1ad45bd5f15f53c4047e95582ccf67454752c0ce07c8be55af6e174c1",
162
+ "model/decoder/7/pp_block/mlp/gate_up_proj/model_weight::data": "2ffc16e0e3167c6d8b3729107de75f631eedb1b5a8f55bb275ca31882201b9c3",
163
+ "model/decoder/7/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
164
+ "model/decoder/8/pp_block/attn/o_proj/model_weight::data": "1f03187efd97bd889537901cbad1f21b85442a2515c4904a683dc5f3e7b7ff0f",
165
+ "model/decoder/8/pp_block/attn/qkv_proj/model_weight::data": "8f43cecb18da96994e6baaddc3cbaa8543640bec1ffd941586890793914ecb5b",
166
+ "model/decoder/8/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
167
+ "model/decoder/8/pp_block/mlp/down_proj/model_weight::data": "14dc5cc429030325cb38d3afed0102218ad803f70f4bd4b18ccdea0ebca547b6",
168
+ "model/decoder/8/pp_block/mlp/gate_up_proj/model_weight::data": "ace143a0baf22f44d7c0876d3db0e596836e0b00e6fa0a1adce50df0c7c6a7b3",
169
+ "model/decoder/8/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
170
+ "model/decoder/9/pp_block/attn/o_proj/model_weight::data": "5784e38f0d4b9721b0c6ad0e983789dfbcdaaff0854034b0f022370be0488bb1",
171
+ "model/decoder/9/pp_block/attn/qkv_proj/model_weight::data": "d8dfb6126991c704aa777e707447d3fee9a420a44ffb52fe876150d92770ed4d",
172
+ "model/decoder/9/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
173
+ "model/decoder/9/pp_block/mlp/down_proj/model_weight::data": "47546e1152a79d2b3beed3789d4f7f66400c752e83c4677879c88cb25b30f1ce",
174
+ "model/decoder/9/pp_block/mlp/gate_up_proj/model_weight::data": "d5aacf034300b8b6c2ed5046f8eabda83c5ee1b3c4844c2701a780e2b2af6c18",
175
+ "model/decoder/9/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
176
+ "model/final_layer_norm/pp_block/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
177
+ "model/token_position_embeddings/pp_block/token_embedding/model_weight::data": "ede112d6ea2e218a99f591cd7703606566d37751cf611ad10851acd6d328c077"
178
+ }
179
+ }
init_1.5B_seed43.hash.json ADDED
@@ -0,0 +1,179 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": "/scratch/bvandur1/zhuicon1/checkpoints/_init_1.5B_seed43/0",
3
+ "n_tensor_files": 170,
4
+ "n_tensors": 170,
5
+ "n_parameters": 1504299008,
6
+ "rolling_sha256": "78ab44e2b2ac954ee441ea340e35969c82cf78bf3f2266f3c8cd0590ce3b8aa3",
7
+ "per_tensor": {
8
+ "model/decoder/0/pp_block/attn/o_proj/model_weight::data": "c2807b83e6acd5f44d1446af0db3ae36cba9ebaf0199b2d655e1b1be96faede1",
9
+ "model/decoder/0/pp_block/attn/qkv_proj/model_weight::data": "c8031211aceb7988f08a7b738c610d0ca527ab08130be9cd1b579e50d0b403e3",
10
+ "model/decoder/0/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
11
+ "model/decoder/0/pp_block/mlp/down_proj/model_weight::data": "7ac8f7fc8677e98ed214c529e63b43528039cf22c6713c9d4c9852f9f3d619ff",
12
+ "model/decoder/0/pp_block/mlp/gate_up_proj/model_weight::data": "cd4ec9ca647982e51765dfdfa3eea574d72cb7376dbf6481e76ad194bfe9cbbd",
13
+ "model/decoder/0/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
14
+ "model/decoder/1/pp_block/attn/o_proj/model_weight::data": "acd56f54cbe4c4b7cde6b69a2c387905d858756ad420ff3c3e720948bf2473d7",
15
+ "model/decoder/1/pp_block/attn/qkv_proj/model_weight::data": "b1108a617abe083b43e1d0d75fa3bea47084641b371bdbc8d7c85e9c5f5edc36",
16
+ "model/decoder/1/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
17
+ "model/decoder/1/pp_block/mlp/down_proj/model_weight::data": "325ed990b3cc9c63cab25f875943b607c36ab9e4d4405c0e3b2227881e922072",
18
+ "model/decoder/1/pp_block/mlp/gate_up_proj/model_weight::data": "784e2a8c76a584de23217d24dc4789117093587c779ccb6b95f42439907cd3d7",
19
+ "model/decoder/1/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
20
+ "model/decoder/10/pp_block/attn/o_proj/model_weight::data": "e08560e360ecd94c5b1f941eb23cd18087383cf1fb93e8301b0b81dedf162fe5",
21
+ "model/decoder/10/pp_block/attn/qkv_proj/model_weight::data": "a78e9d66ef4ad1d6d9ffcd4605e631c0e7767d6b88796e4a7e2c49d0bd3f6abe",
22
+ "model/decoder/10/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
23
+ "model/decoder/10/pp_block/mlp/down_proj/model_weight::data": "182ca4dbfa153690582c924356d9bd388e08f8d9529626e7d327241dc73140ba",
24
+ "model/decoder/10/pp_block/mlp/gate_up_proj/model_weight::data": "2cac872387862a464841e6da6e722879fdbb23942ba4361994a29ac4400d5dad",
25
+ "model/decoder/10/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
26
+ "model/decoder/11/pp_block/attn/o_proj/model_weight::data": "feb7dd1a728f43dcb184f5609ca96258d9a13666cfd74e7ba7241bf5a4dba311",
27
+ "model/decoder/11/pp_block/attn/qkv_proj/model_weight::data": "f540399e06644ec7977273171a740538c8332a13b85de3ca73aaba27e0a5c97d",
28
+ "model/decoder/11/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
29
+ "model/decoder/11/pp_block/mlp/down_proj/model_weight::data": "0389c35f6937b874c668e8ddf45797b7e163dac883ce9783aa40658b457b540c",
30
+ "model/decoder/11/pp_block/mlp/gate_up_proj/model_weight::data": "dc156cfa1eaef064cd9e58ee3b0e9fc1a1ec2548100f5e3fbbacd1b39d688a26",
31
+ "model/decoder/11/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
32
+ "model/decoder/12/pp_block/attn/o_proj/model_weight::data": "1c0aa1f75692fca98ce47527f18f76904446ffc7fdd3bb5461f42b425c856c76",
33
+ "model/decoder/12/pp_block/attn/qkv_proj/model_weight::data": "d26a8b21afff5d0ec13e14859265c250d8602bd4e9117a5bcab09f0765937cfc",
34
+ "model/decoder/12/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
35
+ "model/decoder/12/pp_block/mlp/down_proj/model_weight::data": "541f2ca46606d0f4b580c74c8de1adb59447f984452edb0b0eb058c0e197a937",
36
+ "model/decoder/12/pp_block/mlp/gate_up_proj/model_weight::data": "6fb32a76ff3971e350afa9c5dd0b688300f4572a9017abec407c2577d09dd804",
37
+ "model/decoder/12/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
38
+ "model/decoder/13/pp_block/attn/o_proj/model_weight::data": "8ed2a3533bd25c99d5094c43eab281e2fc40a10787c8af81e840c6027c65082c",
39
+ "model/decoder/13/pp_block/attn/qkv_proj/model_weight::data": "ecf26d862f5352e5d89909d1c029252e8419f8043fc5027cda48bd47d6c53575",
40
+ "model/decoder/13/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
41
+ "model/decoder/13/pp_block/mlp/down_proj/model_weight::data": "ae5de1bce3d1889657a7039d6c777cb56cedcb2f73f3a942cab1b2f76bfa70c8",
42
+ "model/decoder/13/pp_block/mlp/gate_up_proj/model_weight::data": "6b82eea9d2edc2e1e2c77370df3be7c4f349e2884c9742fdfdaaf75cf79c8876",
43
+ "model/decoder/13/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
44
+ "model/decoder/14/pp_block/attn/o_proj/model_weight::data": "59b44e107e02afe2ecb676ddf2ffd7c57284c1c7b9e24bdc77f9e9e7865ef4b6",
45
+ "model/decoder/14/pp_block/attn/qkv_proj/model_weight::data": "771f165a2ea1d8be7b1fa903fd3fdbdafe0c32a7af3401ad4ba5d7989ae26e64",
46
+ "model/decoder/14/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
47
+ "model/decoder/14/pp_block/mlp/down_proj/model_weight::data": "495dc4a15ab3b2b1c26a6f8631ed82bb51e72c5f5eedeb4ab6941e610c29fa02",
48
+ "model/decoder/14/pp_block/mlp/gate_up_proj/model_weight::data": "c1839b1f179a3d03151d5529cc5d94335e41db087dd4b607b857cdb87f4f22fd",
49
+ "model/decoder/14/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
50
+ "model/decoder/15/pp_block/attn/o_proj/model_weight::data": "d2db00d6d78d06c67881fdd419071703496f0a87c4991851b583e42a1b98f890",
51
+ "model/decoder/15/pp_block/attn/qkv_proj/model_weight::data": "3e527b79e449a23220066afd036585f7e6a553ccff540b992df06530e2aa9b15",
52
+ "model/decoder/15/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
53
+ "model/decoder/15/pp_block/mlp/down_proj/model_weight::data": "7c41d24091039b532e8b3653843b13cf1dc3a419ff844252079ce3798ede1ad1",
54
+ "model/decoder/15/pp_block/mlp/gate_up_proj/model_weight::data": "45adf45b1940383db75d5b25349fcb8442c2880923da0ce5a4d447761690deec",
55
+ "model/decoder/15/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
56
+ "model/decoder/16/pp_block/attn/o_proj/model_weight::data": "337a99874e6dd389c8d2c910ef39043dc37c1c47d2ccedda3f1f7f8a2f644174",
57
+ "model/decoder/16/pp_block/attn/qkv_proj/model_weight::data": "5cf1840a5c940a4807c680814825009a6a948a1b08db2a7d1f7021c1e24d663d",
58
+ "model/decoder/16/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
59
+ "model/decoder/16/pp_block/mlp/down_proj/model_weight::data": "b063ac5d482aa1aaefd834a93346e66478c281322e43b08a1e2bda5dc24bf6f4",
60
+ "model/decoder/16/pp_block/mlp/gate_up_proj/model_weight::data": "2c67a2d3268581f21a8c2fe6b32384e03ea6fef8961ef8e8ac698b8791721631",
61
+ "model/decoder/16/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
62
+ "model/decoder/17/pp_block/attn/o_proj/model_weight::data": "e71c5c5a3df0d3b188e224975257248dc5963a202d5433ebf2c2802ef4b0d79d",
63
+ "model/decoder/17/pp_block/attn/qkv_proj/model_weight::data": "c49e38cbd6f247b286743b99d53bb0681ef067b2a14c557668f9a9a28cf46b14",
64
+ "model/decoder/17/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
65
+ "model/decoder/17/pp_block/mlp/down_proj/model_weight::data": "06bbe844c5a26f63c243099a3ba667a383cc0df61c71b346c7e5c271e2f4ae42",
66
+ "model/decoder/17/pp_block/mlp/gate_up_proj/model_weight::data": "99f68e2bbe9e4cd1121923f198144c1f732ddb88cc266843dc70061d80d399fc",
67
+ "model/decoder/17/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
68
+ "model/decoder/18/pp_block/attn/o_proj/model_weight::data": "e82141434ca7e3429e87d8b134e9b6966ee2fbe3c2b4b9eae866bb916061ae07",
69
+ "model/decoder/18/pp_block/attn/qkv_proj/model_weight::data": "f0a86233544e529e7b896d2de72433909eedb92709c501c3200571ae9dd87753",
70
+ "model/decoder/18/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
71
+ "model/decoder/18/pp_block/mlp/down_proj/model_weight::data": "512f1f82b2aea5f1434860fda3968a674e1873a254cf38fbd10b089816e74a68",
72
+ "model/decoder/18/pp_block/mlp/gate_up_proj/model_weight::data": "e75234b65bb49007c96571ef46aac0906f6e1e339cf3584317ae2efc8c6b87e3",
73
+ "model/decoder/18/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
74
+ "model/decoder/19/pp_block/attn/o_proj/model_weight::data": "23a69ed19ae46d3a159ae2fed503b0524a50615d42f00b38fa9669b575e342da",
75
+ "model/decoder/19/pp_block/attn/qkv_proj/model_weight::data": "e302b2b272a32874b5d9d98aa760647f44cf85118d9901e667c10bc2a22e1202",
76
+ "model/decoder/19/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
77
+ "model/decoder/19/pp_block/mlp/down_proj/model_weight::data": "7180f914731be887001af16dac966773494cd56234ef773107ee94eb8849d935",
78
+ "model/decoder/19/pp_block/mlp/gate_up_proj/model_weight::data": "6a1d11f93674479e2be41377eb152a9ac80570309aa9f04f0132fc2585d3d422",
79
+ "model/decoder/19/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
80
+ "model/decoder/2/pp_block/attn/o_proj/model_weight::data": "4016dbec4fe5ab828cea48e098c61c904b2bc7e3f730f3f400fbf672a8be2529",
81
+ "model/decoder/2/pp_block/attn/qkv_proj/model_weight::data": "41b0ed1ed1c7ab623c62002a7e750b9b5f5a0c6a36bf30c302db2ccc9c588e8d",
82
+ "model/decoder/2/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
83
+ "model/decoder/2/pp_block/mlp/down_proj/model_weight::data": "e1c7b29ae2e376524550d13cf752b2979042e72398db5ebebcaedad0ae989ceb",
84
+ "model/decoder/2/pp_block/mlp/gate_up_proj/model_weight::data": "68d439d95fe74c205a31f12e85153cfe7ee8ed33e894a8360293179e02d65750",
85
+ "model/decoder/2/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
86
+ "model/decoder/20/pp_block/attn/o_proj/model_weight::data": "494702d70a8cc5330d1de296833b5f0d0425e3a2133bed926e16fbc72584226f",
87
+ "model/decoder/20/pp_block/attn/qkv_proj/model_weight::data": "3b89515c68cb25590e607c7931202c83e42afab7f18c7517306b9e0430d1e589",
88
+ "model/decoder/20/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
89
+ "model/decoder/20/pp_block/mlp/down_proj/model_weight::data": "4fa215a2a74edc91030ae5e69a50332f441698514075c26a21e3441b101e9cb5",
90
+ "model/decoder/20/pp_block/mlp/gate_up_proj/model_weight::data": "15a87b348781341bb83f5933dd7edaf670b6803a8488f24bc1e3f12b09d39be8",
91
+ "model/decoder/20/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
92
+ "model/decoder/21/pp_block/attn/o_proj/model_weight::data": "155011e8e9ce080d534496d53283c51820e1ea56ee47ad2fb31d8e02ee38f40e",
93
+ "model/decoder/21/pp_block/attn/qkv_proj/model_weight::data": "57454cef978149b19a91440f2e9f7ad1efc60536c8922a351765962b0cf65e07",
94
+ "model/decoder/21/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
95
+ "model/decoder/21/pp_block/mlp/down_proj/model_weight::data": "81d64d72bb557e514f55dbc72ea7916682bf8cefee053b1cf9527ae24202a7ed",
96
+ "model/decoder/21/pp_block/mlp/gate_up_proj/model_weight::data": "cc438f9cad106b90181eef9af2675b9c0ea14bbf0a31dca0272ca18939f27fb4",
97
+ "model/decoder/21/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
98
+ "model/decoder/22/pp_block/attn/o_proj/model_weight::data": "325d6815b2b32d9a0dfecbb853df1bf1bc0aa8d8ddcfc5b3d36582dea6b3ea0a",
99
+ "model/decoder/22/pp_block/attn/qkv_proj/model_weight::data": "27de127170911a5bb8e08e4058c7037c8d240c8b3a0068720719165e26aa7c12",
100
+ "model/decoder/22/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
101
+ "model/decoder/22/pp_block/mlp/down_proj/model_weight::data": "eb1645779f9f0904803fffe22eaff9382b8f2941b55ecf8d39de02e69f22fa5b",
102
+ "model/decoder/22/pp_block/mlp/gate_up_proj/model_weight::data": "85da5269d2bfffb6818834b3002f9738f7a7630973b8e6ec4e4f9e811039d4c1",
103
+ "model/decoder/22/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
104
+ "model/decoder/23/pp_block/attn/o_proj/model_weight::data": "04ace46fd2add16e59fd85958ffe43f5e9e271397f3743d2a7c88230ff20c40b",
105
+ "model/decoder/23/pp_block/attn/qkv_proj/model_weight::data": "2dad180957454fc31c07e6aaa099c8043f7db806bcd3633084a99483c8b9e251",
106
+ "model/decoder/23/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
107
+ "model/decoder/23/pp_block/mlp/down_proj/model_weight::data": "6d05bcb70f922c74f70f3ffd200319909b403319c740615eae7f718d2f8e44e6",
108
+ "model/decoder/23/pp_block/mlp/gate_up_proj/model_weight::data": "0c56ae8f8ed392540c2c983f52e417e2cdeeebf312034c752784fb13b53e0743",
109
+ "model/decoder/23/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
110
+ "model/decoder/24/pp_block/attn/o_proj/model_weight::data": "f2814199736c35b47aca9e1c4f89c6e3f5d4a60385bd213722e131022caa0f60",
111
+ "model/decoder/24/pp_block/attn/qkv_proj/model_weight::data": "23c0df9a081a823541d1580b29fa6526801861849bbd118c2dd8c63283de81f2",
112
+ "model/decoder/24/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
113
+ "model/decoder/24/pp_block/mlp/down_proj/model_weight::data": "11e66b237f007aaa1e531b95f470744d052413020345a97c061d3717a1b33358",
114
+ "model/decoder/24/pp_block/mlp/gate_up_proj/model_weight::data": "29b21af5d0b31b84ecc92c83dd3bfc778019987254d2689a44bda2088f730f72",
115
+ "model/decoder/24/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
116
+ "model/decoder/25/pp_block/attn/o_proj/model_weight::data": "347f1162e59f6a942bbe66c6a1f9ca3397867ba60a4625594eacbfd9a4f8d35c",
117
+ "model/decoder/25/pp_block/attn/qkv_proj/model_weight::data": "5669310bb42b8edbd7c4efe08a992e66620ba6c469209b2f4c57169fb5fd61ff",
118
+ "model/decoder/25/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
119
+ "model/decoder/25/pp_block/mlp/down_proj/model_weight::data": "886719713e818ba73544e8022bb869f0d67579d9014f6987e5fc31aa35212fb2",
120
+ "model/decoder/25/pp_block/mlp/gate_up_proj/model_weight::data": "c88a70a6dff9875d200f90d8f44c6f2f9448dc57d89f4487cd4b6f2195c060aa",
121
+ "model/decoder/25/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
122
+ "model/decoder/26/pp_block/attn/o_proj/model_weight::data": "b49e6e54d76dc4cb6f84190d4734363ffff710edd7a34771e63c207678f7c2e6",
123
+ "model/decoder/26/pp_block/attn/qkv_proj/model_weight::data": "f0521ebf2e0ba5154cdcb2ddfd9c71a93f18b9f0254cedc24357cfa496a67401",
124
+ "model/decoder/26/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
125
+ "model/decoder/26/pp_block/mlp/down_proj/model_weight::data": "733851e505aa85a945bfba93ed9679c021d5c6a7e66ce5eb270d762433f52370",
126
+ "model/decoder/26/pp_block/mlp/gate_up_proj/model_weight::data": "8c1f6f63ad1e48465b74cb068ded060e2b7a89c855ba2d01ba7158132cec5a50",
127
+ "model/decoder/26/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
128
+ "model/decoder/27/pp_block/attn/o_proj/model_weight::data": "95fd0d7fa784eb2eb7bfc957d1d481fd8bfb06ca7bc5efb67ef9ad0e6b6dc4f3",
129
+ "model/decoder/27/pp_block/attn/qkv_proj/model_weight::data": "3b76b3a05d19b6275d785df58c82d69950d206117d6f4183d2551b543feae4db",
130
+ "model/decoder/27/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
131
+ "model/decoder/27/pp_block/mlp/down_proj/model_weight::data": "22c81a36daedabe88a9c03316f26a8a329a3087b2ed26d42be5ca45570c6d96e",
132
+ "model/decoder/27/pp_block/mlp/gate_up_proj/model_weight::data": "5d0bca3af659331c7c6091ea398a0ba6b16506008f5f0433f3df03286b09b364",
133
+ "model/decoder/27/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
134
+ "model/decoder/3/pp_block/attn/o_proj/model_weight::data": "ad5a88ebe70188c2f8fb3db4c65cd75bcc7f446b70ac86aa6849929ae0126afb",
135
+ "model/decoder/3/pp_block/attn/qkv_proj/model_weight::data": "b7cfa42b8e38d573957147edc200e4bfccfc20d4ec933629966c56c372ad8667",
136
+ "model/decoder/3/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
137
+ "model/decoder/3/pp_block/mlp/down_proj/model_weight::data": "775e3e6769540789fa81d863678893739971bda033d723b9578059a065aac2c7",
138
+ "model/decoder/3/pp_block/mlp/gate_up_proj/model_weight::data": "186ca8696e39ab8c295aa637e9069839e8b58fc58e4d00ea6edd6a671243079f",
139
+ "model/decoder/3/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
140
+ "model/decoder/4/pp_block/attn/o_proj/model_weight::data": "dc8ee18fdc15fa66c3dfac5e11b2cdb2fb95d215f640b27f769dc10e245e3fed",
141
+ "model/decoder/4/pp_block/attn/qkv_proj/model_weight::data": "d56a4b265ebc1b6023ee02f048ddca0e6edda93ef0fcb9a56dc0893d4e49f54d",
142
+ "model/decoder/4/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
143
+ "model/decoder/4/pp_block/mlp/down_proj/model_weight::data": "45824694ce09624afaa33cef9481661f81a3a9a5e510078dcb16ad4104b48d5e",
144
+ "model/decoder/4/pp_block/mlp/gate_up_proj/model_weight::data": "a4b51de60659b051fb7f6b70b526274be5a0f503a11d678baf217b3243d200ed",
145
+ "model/decoder/4/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
146
+ "model/decoder/5/pp_block/attn/o_proj/model_weight::data": "6b4be274b50bca0eab9acacfa10277f0e625190c9641f6d17c7ce76be2c8cb42",
147
+ "model/decoder/5/pp_block/attn/qkv_proj/model_weight::data": "faa173cd53b0131ea22a66e71496a26ad9dca26cefad9147e02f3c3beff8c02f",
148
+ "model/decoder/5/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
149
+ "model/decoder/5/pp_block/mlp/down_proj/model_weight::data": "43d32027a2e4948efd14b4385f8c5f959b3f5578ce410b6b375cac5ddd758da5",
150
+ "model/decoder/5/pp_block/mlp/gate_up_proj/model_weight::data": "4900b94f1902d07834fbddd07294b578b17570e894b10ccb3579158943d099ff",
151
+ "model/decoder/5/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
152
+ "model/decoder/6/pp_block/attn/o_proj/model_weight::data": "56a9e3d1c2a0bab8cb302a4f37a0acb11ba87f46c20ca191f7b29ddc57ea66d7",
153
+ "model/decoder/6/pp_block/attn/qkv_proj/model_weight::data": "e2b48c2d89da91896597c4ef6333ff032eeef43aaa328732d93fefa6a700447e",
154
+ "model/decoder/6/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
155
+ "model/decoder/6/pp_block/mlp/down_proj/model_weight::data": "64d2de382a0a8bf07b738394cc6763e9d27a2a72a1a151ec30cf3256bc8d1c49",
156
+ "model/decoder/6/pp_block/mlp/gate_up_proj/model_weight::data": "41e196aa43220ce3d39bd14f09082844d22818a1fdbe7b4aade98cb3868ffdea",
157
+ "model/decoder/6/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
158
+ "model/decoder/7/pp_block/attn/o_proj/model_weight::data": "9d32f26639a33725aa8d8a157fab81885785ae28c94c5cd7d0e7e454dc842cd4",
159
+ "model/decoder/7/pp_block/attn/qkv_proj/model_weight::data": "886eb8a0909efdd6182889e54ac520e0ccc08a66372c6768680792d7ed34de43",
160
+ "model/decoder/7/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
161
+ "model/decoder/7/pp_block/mlp/down_proj/model_weight::data": "8697cf4f0a842df67c4212dc113179a65d3352000df828fbc3e050c91269f50f",
162
+ "model/decoder/7/pp_block/mlp/gate_up_proj/model_weight::data": "10f541449defb034aa5e6f8a98db3f363b3985ff91c75e32a9330e527c673ccd",
163
+ "model/decoder/7/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
164
+ "model/decoder/8/pp_block/attn/o_proj/model_weight::data": "8160abe7386261cb28fead6d72e3a04e93230263aaa12578d2097a753dfc65af",
165
+ "model/decoder/8/pp_block/attn/qkv_proj/model_weight::data": "c263cb00335a9c1c590578e3dd1fcbc839c769c97a4563d76479c3960b24b885",
166
+ "model/decoder/8/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
167
+ "model/decoder/8/pp_block/mlp/down_proj/model_weight::data": "62aa40c095d6fe2eba8d7708dd4795b483378e7d9451d1bb213f3aabb835f3d4",
168
+ "model/decoder/8/pp_block/mlp/gate_up_proj/model_weight::data": "b00ce67bed2369511651c3324aa1db745ee655ea82bc3473c203ddb24b016801",
169
+ "model/decoder/8/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
170
+ "model/decoder/9/pp_block/attn/o_proj/model_weight::data": "11c5f1e5abeff0d12f8ef4b6b91c23e07ad7345acbd51d44118678ea898aca58",
171
+ "model/decoder/9/pp_block/attn/qkv_proj/model_weight::data": "a1b605cceaefbe7e069da94eece53c60f27481b3911369cfcc06c108962b4902",
172
+ "model/decoder/9/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
173
+ "model/decoder/9/pp_block/mlp/down_proj/model_weight::data": "1154c60201cd9a68f04ca1bc8fbe956889c39bcb23dadadf9f5632033222d6b8",
174
+ "model/decoder/9/pp_block/mlp/gate_up_proj/model_weight::data": "1794972e18ba7f4b3252ecc372936509659dc2d8da4eb5f4256de9826d9e70f0",
175
+ "model/decoder/9/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
176
+ "model/final_layer_norm/pp_block/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
177
+ "model/token_position_embeddings/pp_block/token_embedding/model_weight::data": "d147efdaa7cdf1718c002f78c05d28fc0a8ce177da3e47b4afb23bb403819c5f"
178
+ }
179
+ }
init_1.5B_seed44.hash.json ADDED
@@ -0,0 +1,179 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": "/scratch/bvandur1/zhuicon1/checkpoints/_init_1.5B_seed44/0",
3
+ "n_tensor_files": 170,
4
+ "n_tensors": 170,
5
+ "n_parameters": 1504299008,
6
+ "rolling_sha256": "a967df1cae0538c63bf1be412d6e3db0082e75936680f98b11b9a84d49642247",
7
+ "per_tensor": {
8
+ "model/decoder/0/pp_block/attn/o_proj/model_weight::data": "7ad0f0438cfd73c1362a91ad8924ca71f799fc60f52f141da27f3dedd4baf02d",
9
+ "model/decoder/0/pp_block/attn/qkv_proj/model_weight::data": "c35360e0dc1484f66f5b040c8c1ced0b5eb03502031e669bcde35c7337041ce8",
10
+ "model/decoder/0/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
11
+ "model/decoder/0/pp_block/mlp/down_proj/model_weight::data": "b704fca9a7cc54247a61af96cfeb682f821e0a31b5d4c0f92887333a470552f1",
12
+ "model/decoder/0/pp_block/mlp/gate_up_proj/model_weight::data": "f7bd631e8fdd61c813056c45fa7f0664b72d3be059841c49a33ef9a94176ace6",
13
+ "model/decoder/0/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
14
+ "model/decoder/1/pp_block/attn/o_proj/model_weight::data": "1bc713658dcb980b531096218ee3338dd4f6b3c69393423a682c8ca87d489577",
15
+ "model/decoder/1/pp_block/attn/qkv_proj/model_weight::data": "8e78b500cb35c7a90f335084983e5adaec28bfc1f1dc5f1393b21d5353bf5fe1",
16
+ "model/decoder/1/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
17
+ "model/decoder/1/pp_block/mlp/down_proj/model_weight::data": "aebb2cf35f92244489a3093aad459a43b505f14df044c3f2018ef6aa32a02d36",
18
+ "model/decoder/1/pp_block/mlp/gate_up_proj/model_weight::data": "b8cd75b9ca7513aef3ccf78c435cbefd4e301847d80cafe217b42ba46c4b91c8",
19
+ "model/decoder/1/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
20
+ "model/decoder/10/pp_block/attn/o_proj/model_weight::data": "323b4076980383e8f694a457cca296068d68afb855be5ad9a02aef7095c62459",
21
+ "model/decoder/10/pp_block/attn/qkv_proj/model_weight::data": "f538aa156c1dfcdff9d445c6e01e2312c555cec9bed144e9007384b04ee25259",
22
+ "model/decoder/10/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
23
+ "model/decoder/10/pp_block/mlp/down_proj/model_weight::data": "4d72a70e618fe4a06d734d497992d525cc278c65ee3a97f4585dac5cec3d3d03",
24
+ "model/decoder/10/pp_block/mlp/gate_up_proj/model_weight::data": "9c62b250a78f8f776b3de1df2333164147393a72c687c649299c6595874e2097",
25
+ "model/decoder/10/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
26
+ "model/decoder/11/pp_block/attn/o_proj/model_weight::data": "cd3b1e22c3fcc1d057feab1da22b566bd5d6db90b69de5834c02905b9f36ff5d",
27
+ "model/decoder/11/pp_block/attn/qkv_proj/model_weight::data": "64106caa0d87e7990c24c4079c687d66caf56ea2050e545c9d749eebc0e247bf",
28
+ "model/decoder/11/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
29
+ "model/decoder/11/pp_block/mlp/down_proj/model_weight::data": "f0629e12d90bba8afcca6a2768e90277dee5cd9478a4cabc8e5ffe37c6b761b6",
30
+ "model/decoder/11/pp_block/mlp/gate_up_proj/model_weight::data": "a5cfa7fec50ab46bf372884601f8fea6733b286c6a0d25cccc1ae0ec702da260",
31
+ "model/decoder/11/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
32
+ "model/decoder/12/pp_block/attn/o_proj/model_weight::data": "e1a0e4c576a733ce1e0b01d8a1c4bf5771ff2fa07f83f7b67d067662cb68740b",
33
+ "model/decoder/12/pp_block/attn/qkv_proj/model_weight::data": "ce65968c6067cf6fb47e81927fef7e8d568485e59f835f05997a56ad82b32b69",
34
+ "model/decoder/12/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
35
+ "model/decoder/12/pp_block/mlp/down_proj/model_weight::data": "830d89b2856f7f6b0546fc7adc64cc742f718de7ed5f6c2de723902e8098b092",
36
+ "model/decoder/12/pp_block/mlp/gate_up_proj/model_weight::data": "a65aead09997793f193104f574a0b1513d410c002eff08e5419e7e0222872cbc",
37
+ "model/decoder/12/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
38
+ "model/decoder/13/pp_block/attn/o_proj/model_weight::data": "b2b17582268847816f1d306859261e7bdf90505afebe91b7b5c263111d8fc703",
39
+ "model/decoder/13/pp_block/attn/qkv_proj/model_weight::data": "95c026350f5d6cd4ab4b06d20ce5141c15cc1231f88e94f894381876ee1f060c",
40
+ "model/decoder/13/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
41
+ "model/decoder/13/pp_block/mlp/down_proj/model_weight::data": "8e083a9fa672899e6b7b0ad11dfd99318238e9bf3c651febb8d77b5999568402",
42
+ "model/decoder/13/pp_block/mlp/gate_up_proj/model_weight::data": "0d00aa8579cc11d63c07d5e8e18ab04584cec884fa44f35fd6874d1b4583a845",
43
+ "model/decoder/13/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
44
+ "model/decoder/14/pp_block/attn/o_proj/model_weight::data": "4905fe778e42b69e6a9d1dc3cc67f35b0842a228ce9c2133bd140b1fd2bb78b6",
45
+ "model/decoder/14/pp_block/attn/qkv_proj/model_weight::data": "9daa3a69df8d25bba07efa4bd85523f913e41677da0e240c2583cc8032afac44",
46
+ "model/decoder/14/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
47
+ "model/decoder/14/pp_block/mlp/down_proj/model_weight::data": "c89c0d503acde32a336db10e4c599d28531dd6de2dac7ad363dbf2f3060054cb",
48
+ "model/decoder/14/pp_block/mlp/gate_up_proj/model_weight::data": "f334de349c44dccace952ab3676a7de0e0ff0a32100ed9a09b2958c517b724ad",
49
+ "model/decoder/14/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
50
+ "model/decoder/15/pp_block/attn/o_proj/model_weight::data": "794ef200085dbe84e2657b86907be048986b4dd104511301a171fa468f824c2b",
51
+ "model/decoder/15/pp_block/attn/qkv_proj/model_weight::data": "e7fc4e7c47a1c11d30c8b8fb8b87aa7fadc93e7b73ca630ac085abf64a1122a0",
52
+ "model/decoder/15/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
53
+ "model/decoder/15/pp_block/mlp/down_proj/model_weight::data": "12dbe8106118a022c58b5b54a1ddd65e78c69223f2c1759a1d2fbf74ac113e27",
54
+ "model/decoder/15/pp_block/mlp/gate_up_proj/model_weight::data": "3f90bd3584dad391dece079a20644c4cc4b35b20b0f10b927d038251122f3791",
55
+ "model/decoder/15/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
56
+ "model/decoder/16/pp_block/attn/o_proj/model_weight::data": "2366e146846bccd2f2121206561366692c2568ea6b33409148caf8679ed83c30",
57
+ "model/decoder/16/pp_block/attn/qkv_proj/model_weight::data": "1e988db4c8e6d1f5a01cdb7e3582a1a4b0259553e42169529b057bd358864dfc",
58
+ "model/decoder/16/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
59
+ "model/decoder/16/pp_block/mlp/down_proj/model_weight::data": "701824f6bb681faa73da3e3c4e265169c3be2fbfdec8651ebf616a9b51e90744",
60
+ "model/decoder/16/pp_block/mlp/gate_up_proj/model_weight::data": "a807eb90b825eb962e6cdfa4969fb92628f8034d4c3d6a0e655f2da2424e15a6",
61
+ "model/decoder/16/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
62
+ "model/decoder/17/pp_block/attn/o_proj/model_weight::data": "f5cecffd43f0caad824bb132741936e1a2aec716bfe9a3eb4600aeb5415ff358",
63
+ "model/decoder/17/pp_block/attn/qkv_proj/model_weight::data": "4f39d71ad90727e45c1f44b46b82319d56cadd680b59ab1dbf4bb6404b61766f",
64
+ "model/decoder/17/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
65
+ "model/decoder/17/pp_block/mlp/down_proj/model_weight::data": "27f4312887a5adfa0a7fed65a7450c320356e57bb4a29e0110305df4019e502a",
66
+ "model/decoder/17/pp_block/mlp/gate_up_proj/model_weight::data": "795ad2009fa65b8b46d74357b2e9aee973fc5f618345b4aacff1147952a91b5c",
67
+ "model/decoder/17/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
68
+ "model/decoder/18/pp_block/attn/o_proj/model_weight::data": "0c645ccdfb8d1c931d2f4a671db80806e46cf65db1d05e37e7ff89dd85f96f97",
69
+ "model/decoder/18/pp_block/attn/qkv_proj/model_weight::data": "ef0af2507bd0b71e7d267dedbbeda29060d1e6c8237ceca1fa57f9ea66c002e4",
70
+ "model/decoder/18/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
71
+ "model/decoder/18/pp_block/mlp/down_proj/model_weight::data": "84e35aa9f9a077f7a7f9d2a8d0398496851661f21fbf3879480e6188ea4eae98",
72
+ "model/decoder/18/pp_block/mlp/gate_up_proj/model_weight::data": "afe1753223abd097263cee7367d3c3e07f520862444142ce8be9bde21df29cdc",
73
+ "model/decoder/18/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
74
+ "model/decoder/19/pp_block/attn/o_proj/model_weight::data": "3bf38eaf71b9fd3fb08736ca4a874cc83424e81814b48fd14e8b2f1807263f55",
75
+ "model/decoder/19/pp_block/attn/qkv_proj/model_weight::data": "066e2f9430e606916b92fc51cabad085f15c03326d0b7262c6b78b5d04c01b37",
76
+ "model/decoder/19/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
77
+ "model/decoder/19/pp_block/mlp/down_proj/model_weight::data": "1db1085052e3c3a71d126e22e09d5e1fd2dc772e4136292d4f7f25246c0f71fd",
78
+ "model/decoder/19/pp_block/mlp/gate_up_proj/model_weight::data": "90e8d48d764c097b2f1f04fb39d76448dd6679afd9cda940b1fa16bd7dcaf096",
79
+ "model/decoder/19/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
80
+ "model/decoder/2/pp_block/attn/o_proj/model_weight::data": "538d83a1f809864da748929849f1760df7d04eaff6601a98f503ff157acccff9",
81
+ "model/decoder/2/pp_block/attn/qkv_proj/model_weight::data": "796415e6df1287754ddffb4317edc70b0255230aec9d6aebdc2b126b76e616e7",
82
+ "model/decoder/2/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
83
+ "model/decoder/2/pp_block/mlp/down_proj/model_weight::data": "ea42d20c47a53bdd8e065a0745ab1d6cfeddf7b8ea1cbf8aaaa96330185f3452",
84
+ "model/decoder/2/pp_block/mlp/gate_up_proj/model_weight::data": "e50d3dff4125001ce9ce2fea1f25865fe850a010ca4e21cd7c611389f656f349",
85
+ "model/decoder/2/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
86
+ "model/decoder/20/pp_block/attn/o_proj/model_weight::data": "53d79f8c16b10b805ab26edd6610a12abf12d6a1e44a4f34f5dfa7e24eaad283",
87
+ "model/decoder/20/pp_block/attn/qkv_proj/model_weight::data": "729f31639af44149068cc1c2c0c9153b38023b2075bad907f28866e1fae6dcda",
88
+ "model/decoder/20/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
89
+ "model/decoder/20/pp_block/mlp/down_proj/model_weight::data": "3a5501ae31f46ec6bd294c56ae2bf3a1ae9fde378b45157eea2dfab7b69e9e6d",
90
+ "model/decoder/20/pp_block/mlp/gate_up_proj/model_weight::data": "7d8995ddfe68778a1981ef7b67eae0f3781e660fe0ce5d16cb15d90aedbcfa80",
91
+ "model/decoder/20/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
92
+ "model/decoder/21/pp_block/attn/o_proj/model_weight::data": "8e02f079cbe9582526bcefbc47283d32438681167c42ecbbf7d949437677c23f",
93
+ "model/decoder/21/pp_block/attn/qkv_proj/model_weight::data": "d774c678f815d144643179310a23fac21862d137d03f95c8b0eb42809af9aee5",
94
+ "model/decoder/21/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
95
+ "model/decoder/21/pp_block/mlp/down_proj/model_weight::data": "9e8f6d735b866eaafe8418242acc69737ed1cf1c48eee49920db0f5c8d4e23f5",
96
+ "model/decoder/21/pp_block/mlp/gate_up_proj/model_weight::data": "e390ca516299c72ac4e7e6ab23ebabc4e361427d5aee01eb85266014c44825be",
97
+ "model/decoder/21/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
98
+ "model/decoder/22/pp_block/attn/o_proj/model_weight::data": "6cb8cbf9b44c6f7dcc884fc5ada89570f9dbf7b8bf3ba70f293c9ebbcebd3519",
99
+ "model/decoder/22/pp_block/attn/qkv_proj/model_weight::data": "3ca086ba09bcf720e72549de64e10638e640378ff558ffb3c4d9627c2c0b90b7",
100
+ "model/decoder/22/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
101
+ "model/decoder/22/pp_block/mlp/down_proj/model_weight::data": "0b6b7aeb936898717920b8df126682d181999b024c61dfc65a8b42261123bc68",
102
+ "model/decoder/22/pp_block/mlp/gate_up_proj/model_weight::data": "ece745bc520b24708b8b095af2b9a63374e70a97011eae323bc0202cc76a5a05",
103
+ "model/decoder/22/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
104
+ "model/decoder/23/pp_block/attn/o_proj/model_weight::data": "6760f4deda22cfd3f67ed24aa5af7027641a8fc392c86b8b3755e88a91ed2b5f",
105
+ "model/decoder/23/pp_block/attn/qkv_proj/model_weight::data": "61057b118c43bf79b4c15ae5761debd903a7d34fdc091f1534b195cdcb3a31e8",
106
+ "model/decoder/23/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
107
+ "model/decoder/23/pp_block/mlp/down_proj/model_weight::data": "c8b73b6a46328da24cf54e7f470c3399ffe86899c0ba73018bbbd08de0de690c",
108
+ "model/decoder/23/pp_block/mlp/gate_up_proj/model_weight::data": "ebeb25d4bbbb8c8767bb16eaa78243d37245bd8e7946a26701f1e62cbe958b6f",
109
+ "model/decoder/23/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
110
+ "model/decoder/24/pp_block/attn/o_proj/model_weight::data": "9c720a14e5bff363caa6044c936b6b84d8e7c4a7af93c5fb1a3dc386438922e9",
111
+ "model/decoder/24/pp_block/attn/qkv_proj/model_weight::data": "8587c1ceca1eb572f06d2ec57dfc91f19fadd0ba3d57525b4d4128984ad4714e",
112
+ "model/decoder/24/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
113
+ "model/decoder/24/pp_block/mlp/down_proj/model_weight::data": "fc89d089dd69b842b3fcd71322d2ae244241ef770afc772473363cf3d7f7c6bb",
114
+ "model/decoder/24/pp_block/mlp/gate_up_proj/model_weight::data": "8f2c2c78d5d49faf13c2430dabea7946fb7f8ad5615cdcb69c146e284a0ada29",
115
+ "model/decoder/24/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
116
+ "model/decoder/25/pp_block/attn/o_proj/model_weight::data": "27216a6b2ddf31eb26c3a8f4a60a22177e526355320d1baaf106d67eddcfdd10",
117
+ "model/decoder/25/pp_block/attn/qkv_proj/model_weight::data": "6f0d8bb2467cd47db28eabd42ad41a2dbcd4d8dcefc5a4d054656e31fd4b1008",
118
+ "model/decoder/25/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
119
+ "model/decoder/25/pp_block/mlp/down_proj/model_weight::data": "3ade118f124b3ac062515684a10904d355ad24da0dba1802f08e6913faf36f66",
120
+ "model/decoder/25/pp_block/mlp/gate_up_proj/model_weight::data": "f54d7f6e5c7a50e95ede116bbdab830beebe116fc4cdcd3f8c6f4368eb9b8a03",
121
+ "model/decoder/25/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
122
+ "model/decoder/26/pp_block/attn/o_proj/model_weight::data": "985efbfa5e72b3a16842a05191bb1af6260a80615858c98d165f882d361738bb",
123
+ "model/decoder/26/pp_block/attn/qkv_proj/model_weight::data": "0e0dc7b4bf6f6e295e78f4f708940a4b917ea6adef4729d22a39d0c1271a5eaf",
124
+ "model/decoder/26/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
125
+ "model/decoder/26/pp_block/mlp/down_proj/model_weight::data": "098a815d9160bef5810313b9be58dd95d344eceadf94ba703d6092b398ea317e",
126
+ "model/decoder/26/pp_block/mlp/gate_up_proj/model_weight::data": "af25b883723e2d34485b3efe8f07d80183186e4a9b40a3c6b483ba069316802d",
127
+ "model/decoder/26/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
128
+ "model/decoder/27/pp_block/attn/o_proj/model_weight::data": "9b9f90faa83b29c7dd6eaab9484921762c6bcab1574e4e73b83dc2dee337404c",
129
+ "model/decoder/27/pp_block/attn/qkv_proj/model_weight::data": "c5ccd416249197f0f31a36a1ea4dccf3b8b6da218c46b8e2d97d4bbad3de674b",
130
+ "model/decoder/27/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
131
+ "model/decoder/27/pp_block/mlp/down_proj/model_weight::data": "fd57c7445c04d0a1cac81163ad9099b033c203465ef0ae245af4882a78ca7d6c",
132
+ "model/decoder/27/pp_block/mlp/gate_up_proj/model_weight::data": "5ceb8c7066c5a2c423147eb49fbe570ac374d3139c50c8b227393f41124da3ad",
133
+ "model/decoder/27/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
134
+ "model/decoder/3/pp_block/attn/o_proj/model_weight::data": "d01e53b738f1563b935e9f6261a683c5f1e4d512a3f626a17e6947e468ad5c8e",
135
+ "model/decoder/3/pp_block/attn/qkv_proj/model_weight::data": "1f7b931dbb0b89e8225791486b87576e799b7b1eb00dff8e93c759cce843e479",
136
+ "model/decoder/3/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
137
+ "model/decoder/3/pp_block/mlp/down_proj/model_weight::data": "308f62c4afbac95519a53a266967a51d20e300a3603dac852cb5c6c08659f87e",
138
+ "model/decoder/3/pp_block/mlp/gate_up_proj/model_weight::data": "a0f590c874a6d12697304cb5f0a1e36ee109811ef2cd074ea7b4265ed30b2433",
139
+ "model/decoder/3/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
140
+ "model/decoder/4/pp_block/attn/o_proj/model_weight::data": "1051fac6b7add438c8b65a801b55e5b092ac846d4ef8beeacb3c61343212fdb5",
141
+ "model/decoder/4/pp_block/attn/qkv_proj/model_weight::data": "441e71df37c13ab8ca95bb428e22bfc8fb7f4761f57734abaea6f34d3512345a",
142
+ "model/decoder/4/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
143
+ "model/decoder/4/pp_block/mlp/down_proj/model_weight::data": "eee206d7e78846e93a146c8f7aeac694176a3fbf352f1c59159d7b22788778f6",
144
+ "model/decoder/4/pp_block/mlp/gate_up_proj/model_weight::data": "e841852602a501d5a765be9af94ce86e0b09d6f09837b90fb4dde49149320458",
145
+ "model/decoder/4/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
146
+ "model/decoder/5/pp_block/attn/o_proj/model_weight::data": "879b48fcb88b4470f0194eb29498cb1f58a845d8c3ef578df917668652adcdaa",
147
+ "model/decoder/5/pp_block/attn/qkv_proj/model_weight::data": "8454cccc2a6378d2df82be6d11e56fcb3855073506385411316e3a3aa5d17ac2",
148
+ "model/decoder/5/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
149
+ "model/decoder/5/pp_block/mlp/down_proj/model_weight::data": "45a0f3b70c8b7786ae06c8be201d4ba66c02316d195aefe730cd492bd488a86d",
150
+ "model/decoder/5/pp_block/mlp/gate_up_proj/model_weight::data": "e473ecfd4dd227a7ff5b8005718a042acb64a1687c10688a28167a854b87e24c",
151
+ "model/decoder/5/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
152
+ "model/decoder/6/pp_block/attn/o_proj/model_weight::data": "6a5ed05bb220bdb71b6f750d63d8de191d1aea2e0bccbf86b0d03c0090affd22",
153
+ "model/decoder/6/pp_block/attn/qkv_proj/model_weight::data": "3560ef9655e0516b8a815ff7745a7da375a23c531f5e03c58cbb00730f8f89f2",
154
+ "model/decoder/6/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
155
+ "model/decoder/6/pp_block/mlp/down_proj/model_weight::data": "a171fa404b243f25f3e2de672c996e77df28da22c42d2af1482304ad7f21d5c0",
156
+ "model/decoder/6/pp_block/mlp/gate_up_proj/model_weight::data": "3260b8164c7a0ad5e5d74cd59673d4a8dd9732e016702ab449e52fdb637aa5c9",
157
+ "model/decoder/6/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
158
+ "model/decoder/7/pp_block/attn/o_proj/model_weight::data": "a96af4beddfcb7032c4aced884bcb45d8cf7aac637e275e2b03d987b96beab49",
159
+ "model/decoder/7/pp_block/attn/qkv_proj/model_weight::data": "7456d9671a3e01a240e093de14e63804f052c6db6cc7dda36d77403a29d0078f",
160
+ "model/decoder/7/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
161
+ "model/decoder/7/pp_block/mlp/down_proj/model_weight::data": "a50e779d676feaa5d60f8ed4b30bb7f4b692953f241b7df520a650deaeee5b21",
162
+ "model/decoder/7/pp_block/mlp/gate_up_proj/model_weight::data": "9320984fbf305e1ca6e4d7c87b8441b30e066d31d6146a71b26e31e308686854",
163
+ "model/decoder/7/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
164
+ "model/decoder/8/pp_block/attn/o_proj/model_weight::data": "17a76314d1d481ddf6b33531f6466fd977a2cb3c332b05473e7babb65a9d88f9",
165
+ "model/decoder/8/pp_block/attn/qkv_proj/model_weight::data": "33e7ff43bf65460e7729140b2b7246fe388ae1a5e1afd5038feea676f90de21d",
166
+ "model/decoder/8/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
167
+ "model/decoder/8/pp_block/mlp/down_proj/model_weight::data": "9bd0b2d8ca99223e3d27cc322029a7b8be0002cb28e4e44eda1478f9a6839dd6",
168
+ "model/decoder/8/pp_block/mlp/gate_up_proj/model_weight::data": "1a9773f1ccf52bc267945ca5f2912f731418771bd6da9112d1bd6605c579342d",
169
+ "model/decoder/8/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
170
+ "model/decoder/9/pp_block/attn/o_proj/model_weight::data": "dee41ceab248d289011691fc0e2baf0db04973adcf4b8ffd17b416fd5a524030",
171
+ "model/decoder/9/pp_block/attn/qkv_proj/model_weight::data": "c87fe00788d21d7ed0374997eb149161d1948ad01f507257c6e08fac2d18c9b2",
172
+ "model/decoder/9/pp_block/input_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
173
+ "model/decoder/9/pp_block/mlp/down_proj/model_weight::data": "da22bcd1f7d050095b24fe59079552e05b3a238739bb80e729989060552b9518",
174
+ "model/decoder/9/pp_block/mlp/gate_up_proj/model_weight::data": "003133e032615e3b3317bcc0e68f8ae01d61c0767100f06f52cff35f4a932f42",
175
+ "model/decoder/9/pp_block/post_attention_layernorm/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
176
+ "model/final_layer_norm/pp_block/model_weight::data": "eabc5549b69674c7411dec56e3bd58316315794309e62aba5207cfd132be9eff",
177
+ "model/token_position_embeddings/pp_block/token_embedding/model_weight::data": "6c46978ff40720d49fc5a89f0ef9ef51197c75c29d7766f9a2d5417b83261ea2"
178
+ }
179
+ }