Add files using upload-large-folder tool
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +1 -0
- config.yaml +724 -0
- data-indices/rank0.tsv.gz +3 -0
- data-indices/rank2.tsv.gz +3 -0
- data-indices/rank3.tsv.gz +3 -0
- step10000/config.yaml +724 -0
- step10000/model/metadata.json +1 -0
- step10000/optim/metadata.json +0 -0
- step10000/train/rank0.pt +3 -0
- step10000/train/rank1.pt +3 -0
- step10000/train/rank2.pt +3 -0
- step10000/train/rank3.pt +3 -0
- step12000/config.yaml +724 -0
- step12000/model/metadata.json +1 -0
- step12000/optim/metadata.json +0 -0
- step12000/train/rank0.pt +3 -0
- step12000/train/rank1.pt +3 -0
- step12000/train/rank2.pt +3 -0
- step12000/train/rank3.pt +3 -0
- step13954/config.yaml +724 -0
- step13954/model/metadata.json +1 -0
- step13954/optim/metadata.json +0 -0
- step13954/train/rank0.pt +3 -0
- step13954/train/rank1.pt +3 -0
- step13954/train/rank2.pt +3 -0
- step13954/train/rank3.pt +3 -0
- step6000/config.yaml +724 -0
- step6000/model/metadata.json +1 -0
- step6000/optim/metadata.json +0 -0
- step6000/train/rank0.pt +3 -0
- step6000/train/rank1.pt +3 -0
- step6000/train/rank2.pt +3 -0
- step6000/train/rank3.pt +3 -0
- step8000/config.yaml +724 -0
- step8000/model/metadata.json +1 -0
- step8000/optim/metadata.json +0 -0
- step8000/train/rank0.pt +3 -0
- step8000/train/rank1.pt +3 -0
- step8000/train/rank2.pt +3 -0
- step8000/train/rank3.pt +3 -0
- train_data/global_indices.npy +3 -0
- wandb/wandb/debug-internal.log +13 -0
- wandb/wandb/debug.log +26 -0
- wandb/wandb/run-20251219_032809-fhag09gx/files/code/OLMo/scripts/train.py +373 -0
- wandb/wandb/run-20251219_032809-fhag09gx/files/config.yaml +883 -0
- wandb/wandb/run-20251219_032809-fhag09gx/files/output.log +3 -0
- wandb/wandb/run-20251219_032809-fhag09gx/files/requirements.txt +163 -0
- wandb/wandb/run-20251219_032809-fhag09gx/files/wandb-metadata.json +94 -0
- wandb/wandb/run-20251219_032809-fhag09gx/files/wandb-summary.json +0 -0
- wandb/wandb/run-20251219_032809-fhag09gx/logs/debug-internal.log +13 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
wandb/wandb/run-20251219_032809-fhag09gx/files/output.log filter=lfs diff=lfs merge=lfs -text
|
config.yaml
ADDED
|
@@ -0,0 +1,724 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
run_name: olmoe-7b-29b-2ep
|
| 2 |
+
seed: 6198
|
| 3 |
+
epoch: null
|
| 4 |
+
dry_run: false
|
| 5 |
+
model:
|
| 6 |
+
d_model: 2048
|
| 7 |
+
n_heads: 16
|
| 8 |
+
n_kv_heads: null
|
| 9 |
+
clip_qkv: null
|
| 10 |
+
n_layers: 16
|
| 11 |
+
mlp_ratio: 1
|
| 12 |
+
mlp_hidden_size: null
|
| 13 |
+
activation_type: swiglu
|
| 14 |
+
block_type: moe
|
| 15 |
+
block_group_size: 1
|
| 16 |
+
alibi: false
|
| 17 |
+
alibi_bias_max: 8.0
|
| 18 |
+
rope: true
|
| 19 |
+
rope_full_precision: true
|
| 20 |
+
rope_theta: 10000
|
| 21 |
+
flash_attention: true
|
| 22 |
+
attention_dropout: 0.0
|
| 23 |
+
multi_query_attention: null
|
| 24 |
+
attention_layer_norm: true
|
| 25 |
+
residual_dropout: 0.0
|
| 26 |
+
embedding_dropout: 0.0
|
| 27 |
+
embedding_layer_norm: false
|
| 28 |
+
layer_norm_type: rms
|
| 29 |
+
layer_norm_with_affine: true
|
| 30 |
+
layer_norm_eps: 1.0e-05
|
| 31 |
+
attention_layer_norm_with_affine: true
|
| 32 |
+
max_sequence_length: 4096
|
| 33 |
+
include_bias: false
|
| 34 |
+
bias_for_layer_norm: false
|
| 35 |
+
scale_logits: false
|
| 36 |
+
vocab_size: 50280
|
| 37 |
+
embedding_size: 50304
|
| 38 |
+
weight_tying: false
|
| 39 |
+
eos_token_id: 0
|
| 40 |
+
pad_token_id: 1
|
| 41 |
+
init_device: meta
|
| 42 |
+
init_fn: normal
|
| 43 |
+
init_std: 0.02
|
| 44 |
+
init_cutoff_factor: 3.0
|
| 45 |
+
precision: amp_bf16
|
| 46 |
+
moe_num_experts: 64
|
| 47 |
+
moe_top_k: 8
|
| 48 |
+
moe_mlp_impl: sparse
|
| 49 |
+
moe_log_expert_assignment: false
|
| 50 |
+
moe_shared_expert: false
|
| 51 |
+
moe_lbl_in_fp32: false
|
| 52 |
+
moe_loss_weight: 0.01
|
| 53 |
+
moe_zloss_weight: 0.001
|
| 54 |
+
moe_dropless: false
|
| 55 |
+
moe_capacity_factor: 1.25
|
| 56 |
+
himoe_enabled: false
|
| 57 |
+
himoe_n_groups: 4
|
| 58 |
+
himoe_top_k_per_group: 2
|
| 59 |
+
himoe_lambda1: 0.01
|
| 60 |
+
himoe_lambda2: 0.1
|
| 61 |
+
scale_emb_init: false
|
| 62 |
+
emb_init_std: null
|
| 63 |
+
norm_after: false
|
| 64 |
+
optimizer:
|
| 65 |
+
name: adamw
|
| 66 |
+
learning_rate: 0.0004
|
| 67 |
+
weight_decay: 0.1
|
| 68 |
+
betas:
|
| 69 |
+
- 0.9
|
| 70 |
+
- 0.95
|
| 71 |
+
eps: 1.0e-08
|
| 72 |
+
no_decay_norm_and_bias: null
|
| 73 |
+
selective_updates: false
|
| 74 |
+
decay_norm_and_bias: true
|
| 75 |
+
decay_embeddings: true
|
| 76 |
+
metrics_log_interval: 10
|
| 77 |
+
record_update_metrics: false
|
| 78 |
+
scheduler:
|
| 79 |
+
name: cosine_with_warmup
|
| 80 |
+
units: tokens
|
| 81 |
+
t_warmup: 1405296000
|
| 82 |
+
t_max: 58540000000
|
| 83 |
+
alpha_f: 0.1
|
| 84 |
+
grad_clip_warmup_steps: null
|
| 85 |
+
grad_clip_warmup_factor: null
|
| 86 |
+
warmup_min_lr: null
|
| 87 |
+
data:
|
| 88 |
+
paths:
|
| 89 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy
|
| 90 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy
|
| 91 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy
|
| 92 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy
|
| 93 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy
|
| 94 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy
|
| 95 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy
|
| 96 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy
|
| 97 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy
|
| 98 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy
|
| 99 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy
|
| 100 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy
|
| 101 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy
|
| 102 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-13-00000.npy
|
| 103 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-14-00000.npy
|
| 104 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-15-00000.npy
|
| 105 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-000-00000.npy
|
| 106 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-001-00000.npy
|
| 107 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-002-00000.npy
|
| 108 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-003-00000.npy
|
| 109 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-004-00000.npy
|
| 110 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-005-00000.npy
|
| 111 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-006-00000.npy
|
| 112 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-007-00000.npy
|
| 113 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-008-00000.npy
|
| 114 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-009-00000.npy
|
| 115 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-010-00000.npy
|
| 116 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-011-00000.npy
|
| 117 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-012-00000.npy
|
| 118 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-013-00000.npy
|
| 119 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy
|
| 120 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy
|
| 121 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy
|
| 122 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy
|
| 123 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy
|
| 124 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy
|
| 125 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy
|
| 126 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy
|
| 127 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy
|
| 128 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy
|
| 129 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy
|
| 130 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy
|
| 131 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy
|
| 132 |
+
memmap_dtype: uint16
|
| 133 |
+
datasets: null
|
| 134 |
+
label_mask_paths: null
|
| 135 |
+
pad_direction: right
|
| 136 |
+
generate_attention_mask: false
|
| 137 |
+
generate_doc_lengths: false
|
| 138 |
+
num_workers: 8
|
| 139 |
+
drop_last: true
|
| 140 |
+
pin_memory: true
|
| 141 |
+
prefetch_factor: 8
|
| 142 |
+
persistent_workers: true
|
| 143 |
+
timeout: 0
|
| 144 |
+
seed: null
|
| 145 |
+
instance_filter:
|
| 146 |
+
repetition_max_period: 13
|
| 147 |
+
repetition_min_period: 1
|
| 148 |
+
repetition_max_count: 32
|
| 149 |
+
restore_dataloader: true
|
| 150 |
+
fast_forward_batches: null
|
| 151 |
+
evaluators:
|
| 152 |
+
- label: all-small-ppl-validation
|
| 153 |
+
type: lm
|
| 154 |
+
data:
|
| 155 |
+
paths: null
|
| 156 |
+
memmap_dtype: uint16
|
| 157 |
+
datasets:
|
| 158 |
+
c4_en-validation:
|
| 159 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/c4_en/val/part-0-00000.npy
|
| 160 |
+
dolma_books-validation:
|
| 161 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_books/val/part-0-00000.npy
|
| 162 |
+
dolma_common-crawl-validation:
|
| 163 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_common-crawl/val/part-0-00000.npy
|
| 164 |
+
dolma_pes2o-validation:
|
| 165 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_pes2o/val/part-0-00000.npy
|
| 166 |
+
dolma_reddit-validation:
|
| 167 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_reddit/val/part-0-00000.npy
|
| 168 |
+
dolma_stack-validation:
|
| 169 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_stack/val/part-0-00000.npy
|
| 170 |
+
dolma_wiki-validation:
|
| 171 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_wiki/val/part-0-00000.npy
|
| 172 |
+
ice-validation:
|
| 173 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/ice/val/part-0-00000.npy
|
| 174 |
+
m2d2_s2orc-validation:
|
| 175 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/m2d2_s2orc/val/part-0-00000.npy
|
| 176 |
+
pile-validation:
|
| 177 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/pile/val/part-0-00000.npy
|
| 178 |
+
wikitext_103-validation:
|
| 179 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/wikitext_103/val/part-0-00000.npy
|
| 180 |
+
label_mask_paths: null
|
| 181 |
+
pad_direction: right
|
| 182 |
+
generate_attention_mask: false
|
| 183 |
+
generate_doc_lengths: false
|
| 184 |
+
num_workers: 0
|
| 185 |
+
drop_last: true
|
| 186 |
+
pin_memory: false
|
| 187 |
+
prefetch_factor: null
|
| 188 |
+
persistent_workers: false
|
| 189 |
+
timeout: 0
|
| 190 |
+
seed: null
|
| 191 |
+
instance_filter: null
|
| 192 |
+
device_eval_batch_size: null
|
| 193 |
+
subset_num_batches: null
|
| 194 |
+
- label: piqa
|
| 195 |
+
type: downstream
|
| 196 |
+
data:
|
| 197 |
+
paths: null
|
| 198 |
+
memmap_dtype: uint16
|
| 199 |
+
datasets: null
|
| 200 |
+
label_mask_paths: null
|
| 201 |
+
pad_direction: right
|
| 202 |
+
generate_attention_mask: false
|
| 203 |
+
generate_doc_lengths: false
|
| 204 |
+
num_workers: 0
|
| 205 |
+
drop_last: false
|
| 206 |
+
pin_memory: false
|
| 207 |
+
prefetch_factor: null
|
| 208 |
+
persistent_workers: false
|
| 209 |
+
timeout: 0
|
| 210 |
+
seed: null
|
| 211 |
+
instance_filter: null
|
| 212 |
+
device_eval_batch_size: null
|
| 213 |
+
subset_num_batches: null
|
| 214 |
+
- label: hellaswag
|
| 215 |
+
type: downstream
|
| 216 |
+
data:
|
| 217 |
+
paths: null
|
| 218 |
+
memmap_dtype: uint16
|
| 219 |
+
datasets: null
|
| 220 |
+
label_mask_paths: null
|
| 221 |
+
pad_direction: right
|
| 222 |
+
generate_attention_mask: false
|
| 223 |
+
generate_doc_lengths: false
|
| 224 |
+
num_workers: 0
|
| 225 |
+
drop_last: false
|
| 226 |
+
pin_memory: false
|
| 227 |
+
prefetch_factor: null
|
| 228 |
+
persistent_workers: false
|
| 229 |
+
timeout: 0
|
| 230 |
+
seed: null
|
| 231 |
+
instance_filter: null
|
| 232 |
+
device_eval_batch_size: null
|
| 233 |
+
subset_num_batches: null
|
| 234 |
+
- label: winogrande
|
| 235 |
+
type: downstream
|
| 236 |
+
data:
|
| 237 |
+
paths: null
|
| 238 |
+
memmap_dtype: uint16
|
| 239 |
+
datasets: null
|
| 240 |
+
label_mask_paths: null
|
| 241 |
+
pad_direction: right
|
| 242 |
+
generate_attention_mask: false
|
| 243 |
+
generate_doc_lengths: false
|
| 244 |
+
num_workers: 0
|
| 245 |
+
drop_last: false
|
| 246 |
+
pin_memory: false
|
| 247 |
+
prefetch_factor: null
|
| 248 |
+
persistent_workers: false
|
| 249 |
+
timeout: 0
|
| 250 |
+
seed: null
|
| 251 |
+
instance_filter: null
|
| 252 |
+
device_eval_batch_size: null
|
| 253 |
+
subset_num_batches: null
|
| 254 |
+
- label: openbook_qa
|
| 255 |
+
type: downstream
|
| 256 |
+
data:
|
| 257 |
+
paths: null
|
| 258 |
+
memmap_dtype: uint16
|
| 259 |
+
datasets: null
|
| 260 |
+
label_mask_paths: null
|
| 261 |
+
pad_direction: right
|
| 262 |
+
generate_attention_mask: false
|
| 263 |
+
generate_doc_lengths: false
|
| 264 |
+
num_workers: 0
|
| 265 |
+
drop_last: false
|
| 266 |
+
pin_memory: false
|
| 267 |
+
prefetch_factor: null
|
| 268 |
+
persistent_workers: false
|
| 269 |
+
timeout: 0
|
| 270 |
+
seed: null
|
| 271 |
+
instance_filter: null
|
| 272 |
+
device_eval_batch_size: null
|
| 273 |
+
subset_num_batches: null
|
| 274 |
+
- label: boolq
|
| 275 |
+
type: downstream
|
| 276 |
+
data:
|
| 277 |
+
paths: null
|
| 278 |
+
memmap_dtype: uint16
|
| 279 |
+
datasets: null
|
| 280 |
+
label_mask_paths: null
|
| 281 |
+
pad_direction: right
|
| 282 |
+
generate_attention_mask: false
|
| 283 |
+
generate_doc_lengths: false
|
| 284 |
+
num_workers: 0
|
| 285 |
+
drop_last: false
|
| 286 |
+
pin_memory: false
|
| 287 |
+
prefetch_factor: null
|
| 288 |
+
persistent_workers: false
|
| 289 |
+
timeout: 0
|
| 290 |
+
seed: null
|
| 291 |
+
instance_filter: null
|
| 292 |
+
device_eval_batch_size: null
|
| 293 |
+
subset_num_batches: null
|
| 294 |
+
- label: sciq
|
| 295 |
+
type: downstream
|
| 296 |
+
data:
|
| 297 |
+
paths: null
|
| 298 |
+
memmap_dtype: uint16
|
| 299 |
+
datasets: null
|
| 300 |
+
label_mask_paths: null
|
| 301 |
+
pad_direction: right
|
| 302 |
+
generate_attention_mask: false
|
| 303 |
+
generate_doc_lengths: false
|
| 304 |
+
num_workers: 0
|
| 305 |
+
drop_last: false
|
| 306 |
+
pin_memory: false
|
| 307 |
+
prefetch_factor: null
|
| 308 |
+
persistent_workers: false
|
| 309 |
+
timeout: 0
|
| 310 |
+
seed: null
|
| 311 |
+
instance_filter: null
|
| 312 |
+
device_eval_batch_size: null
|
| 313 |
+
subset_num_batches: null
|
| 314 |
+
- label: arc_easy
|
| 315 |
+
type: downstream
|
| 316 |
+
data:
|
| 317 |
+
paths: null
|
| 318 |
+
memmap_dtype: uint16
|
| 319 |
+
datasets: null
|
| 320 |
+
label_mask_paths: null
|
| 321 |
+
pad_direction: right
|
| 322 |
+
generate_attention_mask: false
|
| 323 |
+
generate_doc_lengths: false
|
| 324 |
+
num_workers: 0
|
| 325 |
+
drop_last: false
|
| 326 |
+
pin_memory: false
|
| 327 |
+
prefetch_factor: null
|
| 328 |
+
persistent_workers: false
|
| 329 |
+
timeout: 0
|
| 330 |
+
seed: null
|
| 331 |
+
instance_filter: null
|
| 332 |
+
device_eval_batch_size: null
|
| 333 |
+
subset_num_batches: null
|
| 334 |
+
- label: arc_challenge
|
| 335 |
+
type: downstream
|
| 336 |
+
data:
|
| 337 |
+
paths: null
|
| 338 |
+
memmap_dtype: uint16
|
| 339 |
+
datasets: null
|
| 340 |
+
label_mask_paths: null
|
| 341 |
+
pad_direction: right
|
| 342 |
+
generate_attention_mask: false
|
| 343 |
+
generate_doc_lengths: false
|
| 344 |
+
num_workers: 0
|
| 345 |
+
drop_last: false
|
| 346 |
+
pin_memory: false
|
| 347 |
+
prefetch_factor: null
|
| 348 |
+
persistent_workers: false
|
| 349 |
+
timeout: 0
|
| 350 |
+
seed: null
|
| 351 |
+
instance_filter: null
|
| 352 |
+
device_eval_batch_size: null
|
| 353 |
+
subset_num_batches: null
|
| 354 |
+
- label: copa
|
| 355 |
+
type: downstream
|
| 356 |
+
data:
|
| 357 |
+
paths: null
|
| 358 |
+
memmap_dtype: uint16
|
| 359 |
+
datasets: null
|
| 360 |
+
label_mask_paths: null
|
| 361 |
+
pad_direction: right
|
| 362 |
+
generate_attention_mask: false
|
| 363 |
+
generate_doc_lengths: false
|
| 364 |
+
num_workers: 0
|
| 365 |
+
drop_last: false
|
| 366 |
+
pin_memory: false
|
| 367 |
+
prefetch_factor: null
|
| 368 |
+
persistent_workers: false
|
| 369 |
+
timeout: 0
|
| 370 |
+
seed: null
|
| 371 |
+
instance_filter: null
|
| 372 |
+
device_eval_batch_size: null
|
| 373 |
+
subset_num_batches: null
|
| 374 |
+
- label: commonsense_qa
|
| 375 |
+
type: downstream
|
| 376 |
+
data:
|
| 377 |
+
paths: null
|
| 378 |
+
memmap_dtype: uint16
|
| 379 |
+
datasets: null
|
| 380 |
+
label_mask_paths: null
|
| 381 |
+
pad_direction: right
|
| 382 |
+
generate_attention_mask: false
|
| 383 |
+
generate_doc_lengths: false
|
| 384 |
+
num_workers: 0
|
| 385 |
+
drop_last: false
|
| 386 |
+
pin_memory: false
|
| 387 |
+
prefetch_factor: null
|
| 388 |
+
persistent_workers: false
|
| 389 |
+
timeout: 0
|
| 390 |
+
seed: null
|
| 391 |
+
instance_filter: null
|
| 392 |
+
device_eval_batch_size: null
|
| 393 |
+
subset_num_batches: null
|
| 394 |
+
- label: social_iqa
|
| 395 |
+
type: downstream
|
| 396 |
+
data:
|
| 397 |
+
paths: null
|
| 398 |
+
memmap_dtype: uint16
|
| 399 |
+
datasets: null
|
| 400 |
+
label_mask_paths: null
|
| 401 |
+
pad_direction: right
|
| 402 |
+
generate_attention_mask: false
|
| 403 |
+
generate_doc_lengths: false
|
| 404 |
+
num_workers: 0
|
| 405 |
+
drop_last: false
|
| 406 |
+
pin_memory: false
|
| 407 |
+
prefetch_factor: null
|
| 408 |
+
persistent_workers: false
|
| 409 |
+
timeout: 0
|
| 410 |
+
seed: null
|
| 411 |
+
instance_filter: null
|
| 412 |
+
device_eval_batch_size: null
|
| 413 |
+
subset_num_batches: null
|
| 414 |
+
- label: mmlu_stem_var
|
| 415 |
+
type: downstream
|
| 416 |
+
data:
|
| 417 |
+
paths: null
|
| 418 |
+
memmap_dtype: uint16
|
| 419 |
+
datasets: null
|
| 420 |
+
label_mask_paths: null
|
| 421 |
+
pad_direction: right
|
| 422 |
+
generate_attention_mask: false
|
| 423 |
+
generate_doc_lengths: false
|
| 424 |
+
num_workers: 0
|
| 425 |
+
drop_last: false
|
| 426 |
+
pin_memory: false
|
| 427 |
+
prefetch_factor: null
|
| 428 |
+
persistent_workers: false
|
| 429 |
+
timeout: 0
|
| 430 |
+
seed: null
|
| 431 |
+
instance_filter: null
|
| 432 |
+
device_eval_batch_size: null
|
| 433 |
+
subset_num_batches: null
|
| 434 |
+
- label: mmlu_humanities_var
|
| 435 |
+
type: downstream
|
| 436 |
+
data:
|
| 437 |
+
paths: null
|
| 438 |
+
memmap_dtype: uint16
|
| 439 |
+
datasets: null
|
| 440 |
+
label_mask_paths: null
|
| 441 |
+
pad_direction: right
|
| 442 |
+
generate_attention_mask: false
|
| 443 |
+
generate_doc_lengths: false
|
| 444 |
+
num_workers: 0
|
| 445 |
+
drop_last: false
|
| 446 |
+
pin_memory: false
|
| 447 |
+
prefetch_factor: null
|
| 448 |
+
persistent_workers: false
|
| 449 |
+
timeout: 0
|
| 450 |
+
seed: null
|
| 451 |
+
instance_filter: null
|
| 452 |
+
device_eval_batch_size: null
|
| 453 |
+
subset_num_batches: null
|
| 454 |
+
- label: mmlu_social_sciences_var
|
| 455 |
+
type: downstream
|
| 456 |
+
data:
|
| 457 |
+
paths: null
|
| 458 |
+
memmap_dtype: uint16
|
| 459 |
+
datasets: null
|
| 460 |
+
label_mask_paths: null
|
| 461 |
+
pad_direction: right
|
| 462 |
+
generate_attention_mask: false
|
| 463 |
+
generate_doc_lengths: false
|
| 464 |
+
num_workers: 0
|
| 465 |
+
drop_last: false
|
| 466 |
+
pin_memory: false
|
| 467 |
+
prefetch_factor: null
|
| 468 |
+
persistent_workers: false
|
| 469 |
+
timeout: 0
|
| 470 |
+
seed: null
|
| 471 |
+
instance_filter: null
|
| 472 |
+
device_eval_batch_size: null
|
| 473 |
+
subset_num_batches: null
|
| 474 |
+
- label: mmlu_other_var
|
| 475 |
+
type: downstream
|
| 476 |
+
data:
|
| 477 |
+
paths: null
|
| 478 |
+
memmap_dtype: uint16
|
| 479 |
+
datasets: null
|
| 480 |
+
label_mask_paths: null
|
| 481 |
+
pad_direction: right
|
| 482 |
+
generate_attention_mask: false
|
| 483 |
+
generate_doc_lengths: false
|
| 484 |
+
num_workers: 0
|
| 485 |
+
drop_last: false
|
| 486 |
+
pin_memory: false
|
| 487 |
+
prefetch_factor: null
|
| 488 |
+
persistent_workers: false
|
| 489 |
+
timeout: 0
|
| 490 |
+
seed: null
|
| 491 |
+
instance_filter: null
|
| 492 |
+
device_eval_batch_size: null
|
| 493 |
+
subset_num_batches: null
|
| 494 |
+
- label: mmlu_stem_mc_5shot
|
| 495 |
+
type: downstream
|
| 496 |
+
data:
|
| 497 |
+
paths: null
|
| 498 |
+
memmap_dtype: uint16
|
| 499 |
+
datasets: null
|
| 500 |
+
label_mask_paths: null
|
| 501 |
+
pad_direction: right
|
| 502 |
+
generate_attention_mask: false
|
| 503 |
+
generate_doc_lengths: false
|
| 504 |
+
num_workers: 0
|
| 505 |
+
drop_last: false
|
| 506 |
+
pin_memory: false
|
| 507 |
+
prefetch_factor: null
|
| 508 |
+
persistent_workers: false
|
| 509 |
+
timeout: 0
|
| 510 |
+
seed: null
|
| 511 |
+
instance_filter: null
|
| 512 |
+
device_eval_batch_size: null
|
| 513 |
+
subset_num_batches: null
|
| 514 |
+
- label: mmlu_humanities_mc_5shot
|
| 515 |
+
type: downstream
|
| 516 |
+
data:
|
| 517 |
+
paths: null
|
| 518 |
+
memmap_dtype: uint16
|
| 519 |
+
datasets: null
|
| 520 |
+
label_mask_paths: null
|
| 521 |
+
pad_direction: right
|
| 522 |
+
generate_attention_mask: false
|
| 523 |
+
generate_doc_lengths: false
|
| 524 |
+
num_workers: 0
|
| 525 |
+
drop_last: false
|
| 526 |
+
pin_memory: false
|
| 527 |
+
prefetch_factor: null
|
| 528 |
+
persistent_workers: false
|
| 529 |
+
timeout: 0
|
| 530 |
+
seed: null
|
| 531 |
+
instance_filter: null
|
| 532 |
+
device_eval_batch_size: null
|
| 533 |
+
subset_num_batches: null
|
| 534 |
+
- label: mmlu_social_sciences_mc_5shot
|
| 535 |
+
type: downstream
|
| 536 |
+
data:
|
| 537 |
+
paths: null
|
| 538 |
+
memmap_dtype: uint16
|
| 539 |
+
datasets: null
|
| 540 |
+
label_mask_paths: null
|
| 541 |
+
pad_direction: right
|
| 542 |
+
generate_attention_mask: false
|
| 543 |
+
generate_doc_lengths: false
|
| 544 |
+
num_workers: 0
|
| 545 |
+
drop_last: false
|
| 546 |
+
pin_memory: false
|
| 547 |
+
prefetch_factor: null
|
| 548 |
+
persistent_workers: false
|
| 549 |
+
timeout: 0
|
| 550 |
+
seed: null
|
| 551 |
+
instance_filter: null
|
| 552 |
+
device_eval_batch_size: null
|
| 553 |
+
subset_num_batches: null
|
| 554 |
+
- label: mmlu_other_mc_5shot
|
| 555 |
+
type: downstream
|
| 556 |
+
data:
|
| 557 |
+
paths: null
|
| 558 |
+
memmap_dtype: uint16
|
| 559 |
+
datasets: null
|
| 560 |
+
label_mask_paths: null
|
| 561 |
+
pad_direction: right
|
| 562 |
+
generate_attention_mask: false
|
| 563 |
+
generate_doc_lengths: false
|
| 564 |
+
num_workers: 0
|
| 565 |
+
drop_last: false
|
| 566 |
+
pin_memory: false
|
| 567 |
+
prefetch_factor: null
|
| 568 |
+
persistent_workers: false
|
| 569 |
+
timeout: 0
|
| 570 |
+
seed: null
|
| 571 |
+
instance_filter: null
|
| 572 |
+
device_eval_batch_size: null
|
| 573 |
+
subset_num_batches: null
|
| 574 |
+
- label: mmlu_stem_mc_5shot_test
|
| 575 |
+
type: downstream
|
| 576 |
+
data:
|
| 577 |
+
paths: null
|
| 578 |
+
memmap_dtype: uint16
|
| 579 |
+
datasets: null
|
| 580 |
+
label_mask_paths: null
|
| 581 |
+
pad_direction: right
|
| 582 |
+
generate_attention_mask: false
|
| 583 |
+
generate_doc_lengths: false
|
| 584 |
+
num_workers: 0
|
| 585 |
+
drop_last: false
|
| 586 |
+
pin_memory: false
|
| 587 |
+
prefetch_factor: null
|
| 588 |
+
persistent_workers: false
|
| 589 |
+
timeout: 0
|
| 590 |
+
seed: null
|
| 591 |
+
instance_filter: null
|
| 592 |
+
device_eval_batch_size: null
|
| 593 |
+
subset_num_batches: null
|
| 594 |
+
- label: mmlu_humanities_mc_5shot_test
|
| 595 |
+
type: downstream
|
| 596 |
+
data:
|
| 597 |
+
paths: null
|
| 598 |
+
memmap_dtype: uint16
|
| 599 |
+
datasets: null
|
| 600 |
+
label_mask_paths: null
|
| 601 |
+
pad_direction: right
|
| 602 |
+
generate_attention_mask: false
|
| 603 |
+
generate_doc_lengths: false
|
| 604 |
+
num_workers: 0
|
| 605 |
+
drop_last: false
|
| 606 |
+
pin_memory: false
|
| 607 |
+
prefetch_factor: null
|
| 608 |
+
persistent_workers: false
|
| 609 |
+
timeout: 0
|
| 610 |
+
seed: null
|
| 611 |
+
instance_filter: null
|
| 612 |
+
device_eval_batch_size: null
|
| 613 |
+
subset_num_batches: null
|
| 614 |
+
- label: mmlu_social_sciences_mc_5shot_test
|
| 615 |
+
type: downstream
|
| 616 |
+
data:
|
| 617 |
+
paths: null
|
| 618 |
+
memmap_dtype: uint16
|
| 619 |
+
datasets: null
|
| 620 |
+
label_mask_paths: null
|
| 621 |
+
pad_direction: right
|
| 622 |
+
generate_attention_mask: false
|
| 623 |
+
generate_doc_lengths: false
|
| 624 |
+
num_workers: 0
|
| 625 |
+
drop_last: false
|
| 626 |
+
pin_memory: false
|
| 627 |
+
prefetch_factor: null
|
| 628 |
+
persistent_workers: false
|
| 629 |
+
timeout: 0
|
| 630 |
+
seed: null
|
| 631 |
+
instance_filter: null
|
| 632 |
+
device_eval_batch_size: null
|
| 633 |
+
subset_num_batches: null
|
| 634 |
+
- label: mmlu_other_mc_5shot_test
|
| 635 |
+
type: downstream
|
| 636 |
+
data:
|
| 637 |
+
paths: null
|
| 638 |
+
memmap_dtype: uint16
|
| 639 |
+
datasets: null
|
| 640 |
+
label_mask_paths: null
|
| 641 |
+
pad_direction: right
|
| 642 |
+
generate_attention_mask: false
|
| 643 |
+
generate_doc_lengths: false
|
| 644 |
+
num_workers: 0
|
| 645 |
+
drop_last: false
|
| 646 |
+
pin_memory: false
|
| 647 |
+
prefetch_factor: null
|
| 648 |
+
persistent_workers: false
|
| 649 |
+
timeout: 0
|
| 650 |
+
seed: null
|
| 651 |
+
instance_filter: null
|
| 652 |
+
device_eval_batch_size: null
|
| 653 |
+
subset_num_batches: null
|
| 654 |
+
eval_interval: 2000
|
| 655 |
+
tokenizer:
|
| 656 |
+
identifier: tokenizers/allenai_gpt-neox-olmo-dolma-v1_5.json
|
| 657 |
+
truncate_direction: right
|
| 658 |
+
save_folder: runs/olmoe-7b-29b-2ep
|
| 659 |
+
remote_save_folder: null
|
| 660 |
+
canceled_check_interval: 50
|
| 661 |
+
save_interval: 2000
|
| 662 |
+
save_interval_unsharded: null
|
| 663 |
+
save_interval_ephemeral: null
|
| 664 |
+
save_num_checkpoints_to_keep: 5
|
| 665 |
+
save_num_unsharded_checkpoints_to_keep: -1
|
| 666 |
+
save_overwrite: true
|
| 667 |
+
force_save_unsharded: false
|
| 668 |
+
no_pre_train_checkpoint: true
|
| 669 |
+
load_path: null
|
| 670 |
+
load_path_sharded_checkpointer: null
|
| 671 |
+
try_load_latest_save: false
|
| 672 |
+
reset_optimizer_state: false
|
| 673 |
+
reset_trainer_state: false
|
| 674 |
+
sharded_checkpointer: olmo_core
|
| 675 |
+
new_style_checkpoints: null
|
| 676 |
+
max_duration: 2ep
|
| 677 |
+
global_train_batch_size: 1024
|
| 678 |
+
device_train_batch_size: 256
|
| 679 |
+
device_train_microbatch_size: 2
|
| 680 |
+
device_eval_batch_size: 1
|
| 681 |
+
eval_subset_num_batches: -1
|
| 682 |
+
eval_on_load: false
|
| 683 |
+
device_train_grad_accum: 128
|
| 684 |
+
max_grad_norm: 1.0
|
| 685 |
+
max_grad_norm_ratio: null
|
| 686 |
+
precision: amp_bf16
|
| 687 |
+
wandb:
|
| 688 |
+
project: hiomoe
|
| 689 |
+
entity: bulgakov
|
| 690 |
+
group: 500gb-test
|
| 691 |
+
name: olmoe-7b-29b-2ep
|
| 692 |
+
tags:
|
| 693 |
+
- watching
|
| 694 |
+
log_artifacts: false
|
| 695 |
+
rank_zero_only: true
|
| 696 |
+
log_interval: 1
|
| 697 |
+
speed_monitor:
|
| 698 |
+
window_size: 1
|
| 699 |
+
gpu_flops_available: null
|
| 700 |
+
console_log_interval: 1
|
| 701 |
+
gen1_gc_interval: 1
|
| 702 |
+
compile: null
|
| 703 |
+
distributed_strategy: fsdp
|
| 704 |
+
fsdp:
|
| 705 |
+
use_orig_params: true
|
| 706 |
+
sharding_strategy: FULL_SHARD
|
| 707 |
+
wrapping_strategy: by_block
|
| 708 |
+
precision: mixed
|
| 709 |
+
hybrid_sharding_num_model_replicas: null
|
| 710 |
+
ddp: null
|
| 711 |
+
softmax_auxiliary_loss: false
|
| 712 |
+
auxiliary_loss_multiplier: 0.0001
|
| 713 |
+
time_limit: null
|
| 714 |
+
extra_steps_after_cancel: 10
|
| 715 |
+
early_stopping_factor: null
|
| 716 |
+
save_data_indices: true
|
| 717 |
+
python_profiling: false
|
| 718 |
+
torch_profiling: false
|
| 719 |
+
stop_at: null
|
| 720 |
+
stop_after: null
|
| 721 |
+
activation_checkpointing: fine_grained
|
| 722 |
+
fused_loss: null
|
| 723 |
+
hf_datasets_cache_dir: null
|
| 724 |
+
module_outputs_save_steps: null
|
data-indices/rank0.tsv.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:43c03869cd5c990f496bd28f24ac1dd9d8e0c0e23930e9700614ca9c0fc6ac3b
|
| 3 |
+
size 13343505
|
data-indices/rank2.tsv.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0ce5ace18c2e62b52f7bcf6ef376f2f0e68828d01054c038bbad72add4e1ae24
|
| 3 |
+
size 13342151
|
data-indices/rank3.tsv.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5c85d8a644470edb0afb2d9a3c6db110673cfc173476a7451601834d8ee9b04f
|
| 3 |
+
size 13343526
|
step10000/config.yaml
ADDED
|
@@ -0,0 +1,724 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
run_name: olmoe-7b-29b-2ep
|
| 2 |
+
seed: 6198
|
| 3 |
+
epoch: null
|
| 4 |
+
dry_run: false
|
| 5 |
+
model:
|
| 6 |
+
d_model: 2048
|
| 7 |
+
n_heads: 16
|
| 8 |
+
n_kv_heads: null
|
| 9 |
+
clip_qkv: null
|
| 10 |
+
n_layers: 16
|
| 11 |
+
mlp_ratio: 1
|
| 12 |
+
mlp_hidden_size: null
|
| 13 |
+
activation_type: swiglu
|
| 14 |
+
block_type: moe
|
| 15 |
+
block_group_size: 1
|
| 16 |
+
alibi: false
|
| 17 |
+
alibi_bias_max: 8.0
|
| 18 |
+
rope: true
|
| 19 |
+
rope_full_precision: true
|
| 20 |
+
rope_theta: 10000
|
| 21 |
+
flash_attention: true
|
| 22 |
+
attention_dropout: 0.0
|
| 23 |
+
multi_query_attention: null
|
| 24 |
+
attention_layer_norm: true
|
| 25 |
+
residual_dropout: 0.0
|
| 26 |
+
embedding_dropout: 0.0
|
| 27 |
+
embedding_layer_norm: false
|
| 28 |
+
layer_norm_type: rms
|
| 29 |
+
layer_norm_with_affine: true
|
| 30 |
+
layer_norm_eps: 1.0e-05
|
| 31 |
+
attention_layer_norm_with_affine: true
|
| 32 |
+
max_sequence_length: 4096
|
| 33 |
+
include_bias: false
|
| 34 |
+
bias_for_layer_norm: false
|
| 35 |
+
scale_logits: false
|
| 36 |
+
vocab_size: 50280
|
| 37 |
+
embedding_size: 50304
|
| 38 |
+
weight_tying: false
|
| 39 |
+
eos_token_id: 0
|
| 40 |
+
pad_token_id: 1
|
| 41 |
+
init_device: meta
|
| 42 |
+
init_fn: normal
|
| 43 |
+
init_std: 0.02
|
| 44 |
+
init_cutoff_factor: 3.0
|
| 45 |
+
precision: amp_bf16
|
| 46 |
+
moe_num_experts: 64
|
| 47 |
+
moe_top_k: 8
|
| 48 |
+
moe_mlp_impl: sparse
|
| 49 |
+
moe_log_expert_assignment: false
|
| 50 |
+
moe_shared_expert: false
|
| 51 |
+
moe_lbl_in_fp32: false
|
| 52 |
+
moe_loss_weight: 0.01
|
| 53 |
+
moe_zloss_weight: 0.001
|
| 54 |
+
moe_dropless: false
|
| 55 |
+
moe_capacity_factor: 1.25
|
| 56 |
+
himoe_enabled: false
|
| 57 |
+
himoe_n_groups: 4
|
| 58 |
+
himoe_top_k_per_group: 2
|
| 59 |
+
himoe_lambda1: 0.01
|
| 60 |
+
himoe_lambda2: 0.1
|
| 61 |
+
scale_emb_init: false
|
| 62 |
+
emb_init_std: null
|
| 63 |
+
norm_after: false
|
| 64 |
+
optimizer:
|
| 65 |
+
name: adamw
|
| 66 |
+
learning_rate: 0.0004
|
| 67 |
+
weight_decay: 0.1
|
| 68 |
+
betas:
|
| 69 |
+
- 0.9
|
| 70 |
+
- 0.95
|
| 71 |
+
eps: 1.0e-08
|
| 72 |
+
no_decay_norm_and_bias: null
|
| 73 |
+
selective_updates: false
|
| 74 |
+
decay_norm_and_bias: true
|
| 75 |
+
decay_embeddings: true
|
| 76 |
+
metrics_log_interval: 10
|
| 77 |
+
record_update_metrics: false
|
| 78 |
+
scheduler:
|
| 79 |
+
name: cosine_with_warmup
|
| 80 |
+
units: tokens
|
| 81 |
+
t_warmup: 1405296000
|
| 82 |
+
t_max: 58540000000
|
| 83 |
+
alpha_f: 0.1
|
| 84 |
+
grad_clip_warmup_steps: null
|
| 85 |
+
grad_clip_warmup_factor: null
|
| 86 |
+
warmup_min_lr: null
|
| 87 |
+
data:
|
| 88 |
+
paths:
|
| 89 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy
|
| 90 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy
|
| 91 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy
|
| 92 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy
|
| 93 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy
|
| 94 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy
|
| 95 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy
|
| 96 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy
|
| 97 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy
|
| 98 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy
|
| 99 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy
|
| 100 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy
|
| 101 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy
|
| 102 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-13-00000.npy
|
| 103 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-14-00000.npy
|
| 104 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-15-00000.npy
|
| 105 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-000-00000.npy
|
| 106 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-001-00000.npy
|
| 107 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-002-00000.npy
|
| 108 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-003-00000.npy
|
| 109 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-004-00000.npy
|
| 110 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-005-00000.npy
|
| 111 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-006-00000.npy
|
| 112 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-007-00000.npy
|
| 113 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-008-00000.npy
|
| 114 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-009-00000.npy
|
| 115 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-010-00000.npy
|
| 116 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-011-00000.npy
|
| 117 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-012-00000.npy
|
| 118 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-013-00000.npy
|
| 119 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy
|
| 120 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy
|
| 121 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy
|
| 122 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy
|
| 123 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy
|
| 124 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy
|
| 125 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy
|
| 126 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy
|
| 127 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy
|
| 128 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy
|
| 129 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy
|
| 130 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy
|
| 131 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy
|
| 132 |
+
memmap_dtype: uint16
|
| 133 |
+
datasets: null
|
| 134 |
+
label_mask_paths: null
|
| 135 |
+
pad_direction: right
|
| 136 |
+
generate_attention_mask: false
|
| 137 |
+
generate_doc_lengths: false
|
| 138 |
+
num_workers: 8
|
| 139 |
+
drop_last: true
|
| 140 |
+
pin_memory: true
|
| 141 |
+
prefetch_factor: 8
|
| 142 |
+
persistent_workers: true
|
| 143 |
+
timeout: 0
|
| 144 |
+
seed: null
|
| 145 |
+
instance_filter:
|
| 146 |
+
repetition_max_period: 13
|
| 147 |
+
repetition_min_period: 1
|
| 148 |
+
repetition_max_count: 32
|
| 149 |
+
restore_dataloader: true
|
| 150 |
+
fast_forward_batches: null
|
| 151 |
+
evaluators:
|
| 152 |
+
- label: all-small-ppl-validation
|
| 153 |
+
type: lm
|
| 154 |
+
data:
|
| 155 |
+
paths: null
|
| 156 |
+
memmap_dtype: uint16
|
| 157 |
+
datasets:
|
| 158 |
+
c4_en-validation:
|
| 159 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/c4_en/val/part-0-00000.npy
|
| 160 |
+
dolma_books-validation:
|
| 161 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_books/val/part-0-00000.npy
|
| 162 |
+
dolma_common-crawl-validation:
|
| 163 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_common-crawl/val/part-0-00000.npy
|
| 164 |
+
dolma_pes2o-validation:
|
| 165 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_pes2o/val/part-0-00000.npy
|
| 166 |
+
dolma_reddit-validation:
|
| 167 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_reddit/val/part-0-00000.npy
|
| 168 |
+
dolma_stack-validation:
|
| 169 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_stack/val/part-0-00000.npy
|
| 170 |
+
dolma_wiki-validation:
|
| 171 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_wiki/val/part-0-00000.npy
|
| 172 |
+
ice-validation:
|
| 173 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/ice/val/part-0-00000.npy
|
| 174 |
+
m2d2_s2orc-validation:
|
| 175 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/m2d2_s2orc/val/part-0-00000.npy
|
| 176 |
+
pile-validation:
|
| 177 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/pile/val/part-0-00000.npy
|
| 178 |
+
wikitext_103-validation:
|
| 179 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/wikitext_103/val/part-0-00000.npy
|
| 180 |
+
label_mask_paths: null
|
| 181 |
+
pad_direction: right
|
| 182 |
+
generate_attention_mask: false
|
| 183 |
+
generate_doc_lengths: false
|
| 184 |
+
num_workers: 0
|
| 185 |
+
drop_last: true
|
| 186 |
+
pin_memory: false
|
| 187 |
+
prefetch_factor: null
|
| 188 |
+
persistent_workers: false
|
| 189 |
+
timeout: 0
|
| 190 |
+
seed: null
|
| 191 |
+
instance_filter: null
|
| 192 |
+
device_eval_batch_size: null
|
| 193 |
+
subset_num_batches: null
|
| 194 |
+
- label: piqa
|
| 195 |
+
type: downstream
|
| 196 |
+
data:
|
| 197 |
+
paths: null
|
| 198 |
+
memmap_dtype: uint16
|
| 199 |
+
datasets: null
|
| 200 |
+
label_mask_paths: null
|
| 201 |
+
pad_direction: right
|
| 202 |
+
generate_attention_mask: false
|
| 203 |
+
generate_doc_lengths: false
|
| 204 |
+
num_workers: 0
|
| 205 |
+
drop_last: false
|
| 206 |
+
pin_memory: false
|
| 207 |
+
prefetch_factor: null
|
| 208 |
+
persistent_workers: false
|
| 209 |
+
timeout: 0
|
| 210 |
+
seed: null
|
| 211 |
+
instance_filter: null
|
| 212 |
+
device_eval_batch_size: null
|
| 213 |
+
subset_num_batches: null
|
| 214 |
+
- label: hellaswag
|
| 215 |
+
type: downstream
|
| 216 |
+
data:
|
| 217 |
+
paths: null
|
| 218 |
+
memmap_dtype: uint16
|
| 219 |
+
datasets: null
|
| 220 |
+
label_mask_paths: null
|
| 221 |
+
pad_direction: right
|
| 222 |
+
generate_attention_mask: false
|
| 223 |
+
generate_doc_lengths: false
|
| 224 |
+
num_workers: 0
|
| 225 |
+
drop_last: false
|
| 226 |
+
pin_memory: false
|
| 227 |
+
prefetch_factor: null
|
| 228 |
+
persistent_workers: false
|
| 229 |
+
timeout: 0
|
| 230 |
+
seed: null
|
| 231 |
+
instance_filter: null
|
| 232 |
+
device_eval_batch_size: null
|
| 233 |
+
subset_num_batches: null
|
| 234 |
+
- label: winogrande
|
| 235 |
+
type: downstream
|
| 236 |
+
data:
|
| 237 |
+
paths: null
|
| 238 |
+
memmap_dtype: uint16
|
| 239 |
+
datasets: null
|
| 240 |
+
label_mask_paths: null
|
| 241 |
+
pad_direction: right
|
| 242 |
+
generate_attention_mask: false
|
| 243 |
+
generate_doc_lengths: false
|
| 244 |
+
num_workers: 0
|
| 245 |
+
drop_last: false
|
| 246 |
+
pin_memory: false
|
| 247 |
+
prefetch_factor: null
|
| 248 |
+
persistent_workers: false
|
| 249 |
+
timeout: 0
|
| 250 |
+
seed: null
|
| 251 |
+
instance_filter: null
|
| 252 |
+
device_eval_batch_size: null
|
| 253 |
+
subset_num_batches: null
|
| 254 |
+
- label: openbook_qa
|
| 255 |
+
type: downstream
|
| 256 |
+
data:
|
| 257 |
+
paths: null
|
| 258 |
+
memmap_dtype: uint16
|
| 259 |
+
datasets: null
|
| 260 |
+
label_mask_paths: null
|
| 261 |
+
pad_direction: right
|
| 262 |
+
generate_attention_mask: false
|
| 263 |
+
generate_doc_lengths: false
|
| 264 |
+
num_workers: 0
|
| 265 |
+
drop_last: false
|
| 266 |
+
pin_memory: false
|
| 267 |
+
prefetch_factor: null
|
| 268 |
+
persistent_workers: false
|
| 269 |
+
timeout: 0
|
| 270 |
+
seed: null
|
| 271 |
+
instance_filter: null
|
| 272 |
+
device_eval_batch_size: null
|
| 273 |
+
subset_num_batches: null
|
| 274 |
+
- label: boolq
|
| 275 |
+
type: downstream
|
| 276 |
+
data:
|
| 277 |
+
paths: null
|
| 278 |
+
memmap_dtype: uint16
|
| 279 |
+
datasets: null
|
| 280 |
+
label_mask_paths: null
|
| 281 |
+
pad_direction: right
|
| 282 |
+
generate_attention_mask: false
|
| 283 |
+
generate_doc_lengths: false
|
| 284 |
+
num_workers: 0
|
| 285 |
+
drop_last: false
|
| 286 |
+
pin_memory: false
|
| 287 |
+
prefetch_factor: null
|
| 288 |
+
persistent_workers: false
|
| 289 |
+
timeout: 0
|
| 290 |
+
seed: null
|
| 291 |
+
instance_filter: null
|
| 292 |
+
device_eval_batch_size: null
|
| 293 |
+
subset_num_batches: null
|
| 294 |
+
- label: sciq
|
| 295 |
+
type: downstream
|
| 296 |
+
data:
|
| 297 |
+
paths: null
|
| 298 |
+
memmap_dtype: uint16
|
| 299 |
+
datasets: null
|
| 300 |
+
label_mask_paths: null
|
| 301 |
+
pad_direction: right
|
| 302 |
+
generate_attention_mask: false
|
| 303 |
+
generate_doc_lengths: false
|
| 304 |
+
num_workers: 0
|
| 305 |
+
drop_last: false
|
| 306 |
+
pin_memory: false
|
| 307 |
+
prefetch_factor: null
|
| 308 |
+
persistent_workers: false
|
| 309 |
+
timeout: 0
|
| 310 |
+
seed: null
|
| 311 |
+
instance_filter: null
|
| 312 |
+
device_eval_batch_size: null
|
| 313 |
+
subset_num_batches: null
|
| 314 |
+
- label: arc_easy
|
| 315 |
+
type: downstream
|
| 316 |
+
data:
|
| 317 |
+
paths: null
|
| 318 |
+
memmap_dtype: uint16
|
| 319 |
+
datasets: null
|
| 320 |
+
label_mask_paths: null
|
| 321 |
+
pad_direction: right
|
| 322 |
+
generate_attention_mask: false
|
| 323 |
+
generate_doc_lengths: false
|
| 324 |
+
num_workers: 0
|
| 325 |
+
drop_last: false
|
| 326 |
+
pin_memory: false
|
| 327 |
+
prefetch_factor: null
|
| 328 |
+
persistent_workers: false
|
| 329 |
+
timeout: 0
|
| 330 |
+
seed: null
|
| 331 |
+
instance_filter: null
|
| 332 |
+
device_eval_batch_size: null
|
| 333 |
+
subset_num_batches: null
|
| 334 |
+
- label: arc_challenge
|
| 335 |
+
type: downstream
|
| 336 |
+
data:
|
| 337 |
+
paths: null
|
| 338 |
+
memmap_dtype: uint16
|
| 339 |
+
datasets: null
|
| 340 |
+
label_mask_paths: null
|
| 341 |
+
pad_direction: right
|
| 342 |
+
generate_attention_mask: false
|
| 343 |
+
generate_doc_lengths: false
|
| 344 |
+
num_workers: 0
|
| 345 |
+
drop_last: false
|
| 346 |
+
pin_memory: false
|
| 347 |
+
prefetch_factor: null
|
| 348 |
+
persistent_workers: false
|
| 349 |
+
timeout: 0
|
| 350 |
+
seed: null
|
| 351 |
+
instance_filter: null
|
| 352 |
+
device_eval_batch_size: null
|
| 353 |
+
subset_num_batches: null
|
| 354 |
+
- label: copa
|
| 355 |
+
type: downstream
|
| 356 |
+
data:
|
| 357 |
+
paths: null
|
| 358 |
+
memmap_dtype: uint16
|
| 359 |
+
datasets: null
|
| 360 |
+
label_mask_paths: null
|
| 361 |
+
pad_direction: right
|
| 362 |
+
generate_attention_mask: false
|
| 363 |
+
generate_doc_lengths: false
|
| 364 |
+
num_workers: 0
|
| 365 |
+
drop_last: false
|
| 366 |
+
pin_memory: false
|
| 367 |
+
prefetch_factor: null
|
| 368 |
+
persistent_workers: false
|
| 369 |
+
timeout: 0
|
| 370 |
+
seed: null
|
| 371 |
+
instance_filter: null
|
| 372 |
+
device_eval_batch_size: null
|
| 373 |
+
subset_num_batches: null
|
| 374 |
+
- label: commonsense_qa
|
| 375 |
+
type: downstream
|
| 376 |
+
data:
|
| 377 |
+
paths: null
|
| 378 |
+
memmap_dtype: uint16
|
| 379 |
+
datasets: null
|
| 380 |
+
label_mask_paths: null
|
| 381 |
+
pad_direction: right
|
| 382 |
+
generate_attention_mask: false
|
| 383 |
+
generate_doc_lengths: false
|
| 384 |
+
num_workers: 0
|
| 385 |
+
drop_last: false
|
| 386 |
+
pin_memory: false
|
| 387 |
+
prefetch_factor: null
|
| 388 |
+
persistent_workers: false
|
| 389 |
+
timeout: 0
|
| 390 |
+
seed: null
|
| 391 |
+
instance_filter: null
|
| 392 |
+
device_eval_batch_size: null
|
| 393 |
+
subset_num_batches: null
|
| 394 |
+
- label: social_iqa
|
| 395 |
+
type: downstream
|
| 396 |
+
data:
|
| 397 |
+
paths: null
|
| 398 |
+
memmap_dtype: uint16
|
| 399 |
+
datasets: null
|
| 400 |
+
label_mask_paths: null
|
| 401 |
+
pad_direction: right
|
| 402 |
+
generate_attention_mask: false
|
| 403 |
+
generate_doc_lengths: false
|
| 404 |
+
num_workers: 0
|
| 405 |
+
drop_last: false
|
| 406 |
+
pin_memory: false
|
| 407 |
+
prefetch_factor: null
|
| 408 |
+
persistent_workers: false
|
| 409 |
+
timeout: 0
|
| 410 |
+
seed: null
|
| 411 |
+
instance_filter: null
|
| 412 |
+
device_eval_batch_size: null
|
| 413 |
+
subset_num_batches: null
|
| 414 |
+
- label: mmlu_stem_var
|
| 415 |
+
type: downstream
|
| 416 |
+
data:
|
| 417 |
+
paths: null
|
| 418 |
+
memmap_dtype: uint16
|
| 419 |
+
datasets: null
|
| 420 |
+
label_mask_paths: null
|
| 421 |
+
pad_direction: right
|
| 422 |
+
generate_attention_mask: false
|
| 423 |
+
generate_doc_lengths: false
|
| 424 |
+
num_workers: 0
|
| 425 |
+
drop_last: false
|
| 426 |
+
pin_memory: false
|
| 427 |
+
prefetch_factor: null
|
| 428 |
+
persistent_workers: false
|
| 429 |
+
timeout: 0
|
| 430 |
+
seed: null
|
| 431 |
+
instance_filter: null
|
| 432 |
+
device_eval_batch_size: null
|
| 433 |
+
subset_num_batches: null
|
| 434 |
+
- label: mmlu_humanities_var
|
| 435 |
+
type: downstream
|
| 436 |
+
data:
|
| 437 |
+
paths: null
|
| 438 |
+
memmap_dtype: uint16
|
| 439 |
+
datasets: null
|
| 440 |
+
label_mask_paths: null
|
| 441 |
+
pad_direction: right
|
| 442 |
+
generate_attention_mask: false
|
| 443 |
+
generate_doc_lengths: false
|
| 444 |
+
num_workers: 0
|
| 445 |
+
drop_last: false
|
| 446 |
+
pin_memory: false
|
| 447 |
+
prefetch_factor: null
|
| 448 |
+
persistent_workers: false
|
| 449 |
+
timeout: 0
|
| 450 |
+
seed: null
|
| 451 |
+
instance_filter: null
|
| 452 |
+
device_eval_batch_size: null
|
| 453 |
+
subset_num_batches: null
|
| 454 |
+
- label: mmlu_social_sciences_var
|
| 455 |
+
type: downstream
|
| 456 |
+
data:
|
| 457 |
+
paths: null
|
| 458 |
+
memmap_dtype: uint16
|
| 459 |
+
datasets: null
|
| 460 |
+
label_mask_paths: null
|
| 461 |
+
pad_direction: right
|
| 462 |
+
generate_attention_mask: false
|
| 463 |
+
generate_doc_lengths: false
|
| 464 |
+
num_workers: 0
|
| 465 |
+
drop_last: false
|
| 466 |
+
pin_memory: false
|
| 467 |
+
prefetch_factor: null
|
| 468 |
+
persistent_workers: false
|
| 469 |
+
timeout: 0
|
| 470 |
+
seed: null
|
| 471 |
+
instance_filter: null
|
| 472 |
+
device_eval_batch_size: null
|
| 473 |
+
subset_num_batches: null
|
| 474 |
+
- label: mmlu_other_var
|
| 475 |
+
type: downstream
|
| 476 |
+
data:
|
| 477 |
+
paths: null
|
| 478 |
+
memmap_dtype: uint16
|
| 479 |
+
datasets: null
|
| 480 |
+
label_mask_paths: null
|
| 481 |
+
pad_direction: right
|
| 482 |
+
generate_attention_mask: false
|
| 483 |
+
generate_doc_lengths: false
|
| 484 |
+
num_workers: 0
|
| 485 |
+
drop_last: false
|
| 486 |
+
pin_memory: false
|
| 487 |
+
prefetch_factor: null
|
| 488 |
+
persistent_workers: false
|
| 489 |
+
timeout: 0
|
| 490 |
+
seed: null
|
| 491 |
+
instance_filter: null
|
| 492 |
+
device_eval_batch_size: null
|
| 493 |
+
subset_num_batches: null
|
| 494 |
+
- label: mmlu_stem_mc_5shot
|
| 495 |
+
type: downstream
|
| 496 |
+
data:
|
| 497 |
+
paths: null
|
| 498 |
+
memmap_dtype: uint16
|
| 499 |
+
datasets: null
|
| 500 |
+
label_mask_paths: null
|
| 501 |
+
pad_direction: right
|
| 502 |
+
generate_attention_mask: false
|
| 503 |
+
generate_doc_lengths: false
|
| 504 |
+
num_workers: 0
|
| 505 |
+
drop_last: false
|
| 506 |
+
pin_memory: false
|
| 507 |
+
prefetch_factor: null
|
| 508 |
+
persistent_workers: false
|
| 509 |
+
timeout: 0
|
| 510 |
+
seed: null
|
| 511 |
+
instance_filter: null
|
| 512 |
+
device_eval_batch_size: null
|
| 513 |
+
subset_num_batches: null
|
| 514 |
+
- label: mmlu_humanities_mc_5shot
|
| 515 |
+
type: downstream
|
| 516 |
+
data:
|
| 517 |
+
paths: null
|
| 518 |
+
memmap_dtype: uint16
|
| 519 |
+
datasets: null
|
| 520 |
+
label_mask_paths: null
|
| 521 |
+
pad_direction: right
|
| 522 |
+
generate_attention_mask: false
|
| 523 |
+
generate_doc_lengths: false
|
| 524 |
+
num_workers: 0
|
| 525 |
+
drop_last: false
|
| 526 |
+
pin_memory: false
|
| 527 |
+
prefetch_factor: null
|
| 528 |
+
persistent_workers: false
|
| 529 |
+
timeout: 0
|
| 530 |
+
seed: null
|
| 531 |
+
instance_filter: null
|
| 532 |
+
device_eval_batch_size: null
|
| 533 |
+
subset_num_batches: null
|
| 534 |
+
- label: mmlu_social_sciences_mc_5shot
|
| 535 |
+
type: downstream
|
| 536 |
+
data:
|
| 537 |
+
paths: null
|
| 538 |
+
memmap_dtype: uint16
|
| 539 |
+
datasets: null
|
| 540 |
+
label_mask_paths: null
|
| 541 |
+
pad_direction: right
|
| 542 |
+
generate_attention_mask: false
|
| 543 |
+
generate_doc_lengths: false
|
| 544 |
+
num_workers: 0
|
| 545 |
+
drop_last: false
|
| 546 |
+
pin_memory: false
|
| 547 |
+
prefetch_factor: null
|
| 548 |
+
persistent_workers: false
|
| 549 |
+
timeout: 0
|
| 550 |
+
seed: null
|
| 551 |
+
instance_filter: null
|
| 552 |
+
device_eval_batch_size: null
|
| 553 |
+
subset_num_batches: null
|
| 554 |
+
- label: mmlu_other_mc_5shot
|
| 555 |
+
type: downstream
|
| 556 |
+
data:
|
| 557 |
+
paths: null
|
| 558 |
+
memmap_dtype: uint16
|
| 559 |
+
datasets: null
|
| 560 |
+
label_mask_paths: null
|
| 561 |
+
pad_direction: right
|
| 562 |
+
generate_attention_mask: false
|
| 563 |
+
generate_doc_lengths: false
|
| 564 |
+
num_workers: 0
|
| 565 |
+
drop_last: false
|
| 566 |
+
pin_memory: false
|
| 567 |
+
prefetch_factor: null
|
| 568 |
+
persistent_workers: false
|
| 569 |
+
timeout: 0
|
| 570 |
+
seed: null
|
| 571 |
+
instance_filter: null
|
| 572 |
+
device_eval_batch_size: null
|
| 573 |
+
subset_num_batches: null
|
| 574 |
+
- label: mmlu_stem_mc_5shot_test
|
| 575 |
+
type: downstream
|
| 576 |
+
data:
|
| 577 |
+
paths: null
|
| 578 |
+
memmap_dtype: uint16
|
| 579 |
+
datasets: null
|
| 580 |
+
label_mask_paths: null
|
| 581 |
+
pad_direction: right
|
| 582 |
+
generate_attention_mask: false
|
| 583 |
+
generate_doc_lengths: false
|
| 584 |
+
num_workers: 0
|
| 585 |
+
drop_last: false
|
| 586 |
+
pin_memory: false
|
| 587 |
+
prefetch_factor: null
|
| 588 |
+
persistent_workers: false
|
| 589 |
+
timeout: 0
|
| 590 |
+
seed: null
|
| 591 |
+
instance_filter: null
|
| 592 |
+
device_eval_batch_size: null
|
| 593 |
+
subset_num_batches: null
|
| 594 |
+
- label: mmlu_humanities_mc_5shot_test
|
| 595 |
+
type: downstream
|
| 596 |
+
data:
|
| 597 |
+
paths: null
|
| 598 |
+
memmap_dtype: uint16
|
| 599 |
+
datasets: null
|
| 600 |
+
label_mask_paths: null
|
| 601 |
+
pad_direction: right
|
| 602 |
+
generate_attention_mask: false
|
| 603 |
+
generate_doc_lengths: false
|
| 604 |
+
num_workers: 0
|
| 605 |
+
drop_last: false
|
| 606 |
+
pin_memory: false
|
| 607 |
+
prefetch_factor: null
|
| 608 |
+
persistent_workers: false
|
| 609 |
+
timeout: 0
|
| 610 |
+
seed: null
|
| 611 |
+
instance_filter: null
|
| 612 |
+
device_eval_batch_size: null
|
| 613 |
+
subset_num_batches: null
|
| 614 |
+
- label: mmlu_social_sciences_mc_5shot_test
|
| 615 |
+
type: downstream
|
| 616 |
+
data:
|
| 617 |
+
paths: null
|
| 618 |
+
memmap_dtype: uint16
|
| 619 |
+
datasets: null
|
| 620 |
+
label_mask_paths: null
|
| 621 |
+
pad_direction: right
|
| 622 |
+
generate_attention_mask: false
|
| 623 |
+
generate_doc_lengths: false
|
| 624 |
+
num_workers: 0
|
| 625 |
+
drop_last: false
|
| 626 |
+
pin_memory: false
|
| 627 |
+
prefetch_factor: null
|
| 628 |
+
persistent_workers: false
|
| 629 |
+
timeout: 0
|
| 630 |
+
seed: null
|
| 631 |
+
instance_filter: null
|
| 632 |
+
device_eval_batch_size: null
|
| 633 |
+
subset_num_batches: null
|
| 634 |
+
- label: mmlu_other_mc_5shot_test
|
| 635 |
+
type: downstream
|
| 636 |
+
data:
|
| 637 |
+
paths: null
|
| 638 |
+
memmap_dtype: uint16
|
| 639 |
+
datasets: null
|
| 640 |
+
label_mask_paths: null
|
| 641 |
+
pad_direction: right
|
| 642 |
+
generate_attention_mask: false
|
| 643 |
+
generate_doc_lengths: false
|
| 644 |
+
num_workers: 0
|
| 645 |
+
drop_last: false
|
| 646 |
+
pin_memory: false
|
| 647 |
+
prefetch_factor: null
|
| 648 |
+
persistent_workers: false
|
| 649 |
+
timeout: 0
|
| 650 |
+
seed: null
|
| 651 |
+
instance_filter: null
|
| 652 |
+
device_eval_batch_size: null
|
| 653 |
+
subset_num_batches: null
|
| 654 |
+
eval_interval: 2000
|
| 655 |
+
tokenizer:
|
| 656 |
+
identifier: tokenizers/allenai_gpt-neox-olmo-dolma-v1_5.json
|
| 657 |
+
truncate_direction: right
|
| 658 |
+
save_folder: runs/olmoe-7b-29b-2ep
|
| 659 |
+
remote_save_folder: null
|
| 660 |
+
canceled_check_interval: 50
|
| 661 |
+
save_interval: 2000
|
| 662 |
+
save_interval_unsharded: null
|
| 663 |
+
save_interval_ephemeral: null
|
| 664 |
+
save_num_checkpoints_to_keep: 5
|
| 665 |
+
save_num_unsharded_checkpoints_to_keep: -1
|
| 666 |
+
save_overwrite: true
|
| 667 |
+
force_save_unsharded: false
|
| 668 |
+
no_pre_train_checkpoint: true
|
| 669 |
+
load_path: null
|
| 670 |
+
load_path_sharded_checkpointer: null
|
| 671 |
+
try_load_latest_save: false
|
| 672 |
+
reset_optimizer_state: false
|
| 673 |
+
reset_trainer_state: false
|
| 674 |
+
sharded_checkpointer: olmo_core
|
| 675 |
+
new_style_checkpoints: null
|
| 676 |
+
max_duration: 2ep
|
| 677 |
+
global_train_batch_size: 1024
|
| 678 |
+
device_train_batch_size: 256
|
| 679 |
+
device_train_microbatch_size: 2
|
| 680 |
+
device_eval_batch_size: 1
|
| 681 |
+
eval_subset_num_batches: -1
|
| 682 |
+
eval_on_load: false
|
| 683 |
+
device_train_grad_accum: 128
|
| 684 |
+
max_grad_norm: 1.0
|
| 685 |
+
max_grad_norm_ratio: null
|
| 686 |
+
precision: amp_bf16
|
| 687 |
+
wandb:
|
| 688 |
+
project: hiomoe
|
| 689 |
+
entity: bulgakov
|
| 690 |
+
group: 500gb-test
|
| 691 |
+
name: olmoe-7b-29b-2ep
|
| 692 |
+
tags:
|
| 693 |
+
- watching
|
| 694 |
+
log_artifacts: false
|
| 695 |
+
rank_zero_only: true
|
| 696 |
+
log_interval: 1
|
| 697 |
+
speed_monitor:
|
| 698 |
+
window_size: 1
|
| 699 |
+
gpu_flops_available: null
|
| 700 |
+
console_log_interval: 1
|
| 701 |
+
gen1_gc_interval: 1
|
| 702 |
+
compile: null
|
| 703 |
+
distributed_strategy: fsdp
|
| 704 |
+
fsdp:
|
| 705 |
+
use_orig_params: true
|
| 706 |
+
sharding_strategy: FULL_SHARD
|
| 707 |
+
wrapping_strategy: by_block
|
| 708 |
+
precision: mixed
|
| 709 |
+
hybrid_sharding_num_model_replicas: null
|
| 710 |
+
ddp: null
|
| 711 |
+
softmax_auxiliary_loss: false
|
| 712 |
+
auxiliary_loss_multiplier: 0.0001
|
| 713 |
+
time_limit: null
|
| 714 |
+
extra_steps_after_cancel: 10
|
| 715 |
+
early_stopping_factor: null
|
| 716 |
+
save_data_indices: true
|
| 717 |
+
python_profiling: false
|
| 718 |
+
torch_profiling: false
|
| 719 |
+
stop_at: 13964
|
| 720 |
+
stop_after: null
|
| 721 |
+
activation_checkpointing: fine_grained
|
| 722 |
+
fused_loss: null
|
| 723 |
+
hf_datasets_cache_dir: null
|
| 724 |
+
module_outputs_save_steps: null
|
step10000/model/metadata.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"tensors": {"transformer.wte.weight": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 51511808]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[51511808, 103022592]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.ln_f.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 1024]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[1024, 2048]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.ff_out.weight": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 51510784]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[51510784, 103022592]], "local_shape": null, "global_offset": null}}}}}
|
step10000/optim/metadata.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
step10000/train/rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:686920a2f9baff37b9093b8f45a6955f2803a3a05e09a6c5c3ea873c936927fe
|
| 3 |
+
size 14997
|
step10000/train/rank1.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8447e2cc3ddfd7ab2a4d5b5bd59292ed81f72d899314916fc25bdbaed3d3add0
|
| 3 |
+
size 14997
|
step10000/train/rank2.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:36dfd18a006ae258a9b8ce1cfc74e9827208dfab2c7aa8dd7b4b42fcfb2f0753
|
| 3 |
+
size 14997
|
step10000/train/rank3.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7f16d62720f8ba4ba2cfdce457771cfa447cb32553ed323340bffc98a96407c3
|
| 3 |
+
size 14997
|
step12000/config.yaml
ADDED
|
@@ -0,0 +1,724 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
run_name: olmoe-7b-29b-2ep
|
| 2 |
+
seed: 6198
|
| 3 |
+
epoch: null
|
| 4 |
+
dry_run: false
|
| 5 |
+
model:
|
| 6 |
+
d_model: 2048
|
| 7 |
+
n_heads: 16
|
| 8 |
+
n_kv_heads: null
|
| 9 |
+
clip_qkv: null
|
| 10 |
+
n_layers: 16
|
| 11 |
+
mlp_ratio: 1
|
| 12 |
+
mlp_hidden_size: null
|
| 13 |
+
activation_type: swiglu
|
| 14 |
+
block_type: moe
|
| 15 |
+
block_group_size: 1
|
| 16 |
+
alibi: false
|
| 17 |
+
alibi_bias_max: 8.0
|
| 18 |
+
rope: true
|
| 19 |
+
rope_full_precision: true
|
| 20 |
+
rope_theta: 10000
|
| 21 |
+
flash_attention: true
|
| 22 |
+
attention_dropout: 0.0
|
| 23 |
+
multi_query_attention: null
|
| 24 |
+
attention_layer_norm: true
|
| 25 |
+
residual_dropout: 0.0
|
| 26 |
+
embedding_dropout: 0.0
|
| 27 |
+
embedding_layer_norm: false
|
| 28 |
+
layer_norm_type: rms
|
| 29 |
+
layer_norm_with_affine: true
|
| 30 |
+
layer_norm_eps: 1.0e-05
|
| 31 |
+
attention_layer_norm_with_affine: true
|
| 32 |
+
max_sequence_length: 4096
|
| 33 |
+
include_bias: false
|
| 34 |
+
bias_for_layer_norm: false
|
| 35 |
+
scale_logits: false
|
| 36 |
+
vocab_size: 50280
|
| 37 |
+
embedding_size: 50304
|
| 38 |
+
weight_tying: false
|
| 39 |
+
eos_token_id: 0
|
| 40 |
+
pad_token_id: 1
|
| 41 |
+
init_device: meta
|
| 42 |
+
init_fn: normal
|
| 43 |
+
init_std: 0.02
|
| 44 |
+
init_cutoff_factor: 3.0
|
| 45 |
+
precision: amp_bf16
|
| 46 |
+
moe_num_experts: 64
|
| 47 |
+
moe_top_k: 8
|
| 48 |
+
moe_mlp_impl: sparse
|
| 49 |
+
moe_log_expert_assignment: false
|
| 50 |
+
moe_shared_expert: false
|
| 51 |
+
moe_lbl_in_fp32: false
|
| 52 |
+
moe_loss_weight: 0.01
|
| 53 |
+
moe_zloss_weight: 0.001
|
| 54 |
+
moe_dropless: false
|
| 55 |
+
moe_capacity_factor: 1.25
|
| 56 |
+
himoe_enabled: false
|
| 57 |
+
himoe_n_groups: 4
|
| 58 |
+
himoe_top_k_per_group: 2
|
| 59 |
+
himoe_lambda1: 0.01
|
| 60 |
+
himoe_lambda2: 0.1
|
| 61 |
+
scale_emb_init: false
|
| 62 |
+
emb_init_std: null
|
| 63 |
+
norm_after: false
|
| 64 |
+
optimizer:
|
| 65 |
+
name: adamw
|
| 66 |
+
learning_rate: 0.0004
|
| 67 |
+
weight_decay: 0.1
|
| 68 |
+
betas:
|
| 69 |
+
- 0.9
|
| 70 |
+
- 0.95
|
| 71 |
+
eps: 1.0e-08
|
| 72 |
+
no_decay_norm_and_bias: null
|
| 73 |
+
selective_updates: false
|
| 74 |
+
decay_norm_and_bias: true
|
| 75 |
+
decay_embeddings: true
|
| 76 |
+
metrics_log_interval: 10
|
| 77 |
+
record_update_metrics: false
|
| 78 |
+
scheduler:
|
| 79 |
+
name: cosine_with_warmup
|
| 80 |
+
units: tokens
|
| 81 |
+
t_warmup: 1405296000
|
| 82 |
+
t_max: 58540000000
|
| 83 |
+
alpha_f: 0.1
|
| 84 |
+
grad_clip_warmup_steps: null
|
| 85 |
+
grad_clip_warmup_factor: null
|
| 86 |
+
warmup_min_lr: null
|
| 87 |
+
data:
|
| 88 |
+
paths:
|
| 89 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy
|
| 90 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy
|
| 91 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy
|
| 92 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy
|
| 93 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy
|
| 94 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy
|
| 95 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy
|
| 96 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy
|
| 97 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy
|
| 98 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy
|
| 99 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy
|
| 100 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy
|
| 101 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy
|
| 102 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-13-00000.npy
|
| 103 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-14-00000.npy
|
| 104 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-15-00000.npy
|
| 105 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-000-00000.npy
|
| 106 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-001-00000.npy
|
| 107 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-002-00000.npy
|
| 108 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-003-00000.npy
|
| 109 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-004-00000.npy
|
| 110 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-005-00000.npy
|
| 111 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-006-00000.npy
|
| 112 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-007-00000.npy
|
| 113 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-008-00000.npy
|
| 114 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-009-00000.npy
|
| 115 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-010-00000.npy
|
| 116 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-011-00000.npy
|
| 117 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-012-00000.npy
|
| 118 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-013-00000.npy
|
| 119 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy
|
| 120 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy
|
| 121 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy
|
| 122 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy
|
| 123 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy
|
| 124 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy
|
| 125 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy
|
| 126 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy
|
| 127 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy
|
| 128 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy
|
| 129 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy
|
| 130 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy
|
| 131 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy
|
| 132 |
+
memmap_dtype: uint16
|
| 133 |
+
datasets: null
|
| 134 |
+
label_mask_paths: null
|
| 135 |
+
pad_direction: right
|
| 136 |
+
generate_attention_mask: false
|
| 137 |
+
generate_doc_lengths: false
|
| 138 |
+
num_workers: 8
|
| 139 |
+
drop_last: true
|
| 140 |
+
pin_memory: true
|
| 141 |
+
prefetch_factor: 8
|
| 142 |
+
persistent_workers: true
|
| 143 |
+
timeout: 0
|
| 144 |
+
seed: null
|
| 145 |
+
instance_filter:
|
| 146 |
+
repetition_max_period: 13
|
| 147 |
+
repetition_min_period: 1
|
| 148 |
+
repetition_max_count: 32
|
| 149 |
+
restore_dataloader: true
|
| 150 |
+
fast_forward_batches: null
|
| 151 |
+
evaluators:
|
| 152 |
+
- label: all-small-ppl-validation
|
| 153 |
+
type: lm
|
| 154 |
+
data:
|
| 155 |
+
paths: null
|
| 156 |
+
memmap_dtype: uint16
|
| 157 |
+
datasets:
|
| 158 |
+
c4_en-validation:
|
| 159 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/c4_en/val/part-0-00000.npy
|
| 160 |
+
dolma_books-validation:
|
| 161 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_books/val/part-0-00000.npy
|
| 162 |
+
dolma_common-crawl-validation:
|
| 163 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_common-crawl/val/part-0-00000.npy
|
| 164 |
+
dolma_pes2o-validation:
|
| 165 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_pes2o/val/part-0-00000.npy
|
| 166 |
+
dolma_reddit-validation:
|
| 167 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_reddit/val/part-0-00000.npy
|
| 168 |
+
dolma_stack-validation:
|
| 169 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_stack/val/part-0-00000.npy
|
| 170 |
+
dolma_wiki-validation:
|
| 171 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_wiki/val/part-0-00000.npy
|
| 172 |
+
ice-validation:
|
| 173 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/ice/val/part-0-00000.npy
|
| 174 |
+
m2d2_s2orc-validation:
|
| 175 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/m2d2_s2orc/val/part-0-00000.npy
|
| 176 |
+
pile-validation:
|
| 177 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/pile/val/part-0-00000.npy
|
| 178 |
+
wikitext_103-validation:
|
| 179 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/wikitext_103/val/part-0-00000.npy
|
| 180 |
+
label_mask_paths: null
|
| 181 |
+
pad_direction: right
|
| 182 |
+
generate_attention_mask: false
|
| 183 |
+
generate_doc_lengths: false
|
| 184 |
+
num_workers: 0
|
| 185 |
+
drop_last: true
|
| 186 |
+
pin_memory: false
|
| 187 |
+
prefetch_factor: null
|
| 188 |
+
persistent_workers: false
|
| 189 |
+
timeout: 0
|
| 190 |
+
seed: null
|
| 191 |
+
instance_filter: null
|
| 192 |
+
device_eval_batch_size: null
|
| 193 |
+
subset_num_batches: null
|
| 194 |
+
- label: piqa
|
| 195 |
+
type: downstream
|
| 196 |
+
data:
|
| 197 |
+
paths: null
|
| 198 |
+
memmap_dtype: uint16
|
| 199 |
+
datasets: null
|
| 200 |
+
label_mask_paths: null
|
| 201 |
+
pad_direction: right
|
| 202 |
+
generate_attention_mask: false
|
| 203 |
+
generate_doc_lengths: false
|
| 204 |
+
num_workers: 0
|
| 205 |
+
drop_last: false
|
| 206 |
+
pin_memory: false
|
| 207 |
+
prefetch_factor: null
|
| 208 |
+
persistent_workers: false
|
| 209 |
+
timeout: 0
|
| 210 |
+
seed: null
|
| 211 |
+
instance_filter: null
|
| 212 |
+
device_eval_batch_size: null
|
| 213 |
+
subset_num_batches: null
|
| 214 |
+
- label: hellaswag
|
| 215 |
+
type: downstream
|
| 216 |
+
data:
|
| 217 |
+
paths: null
|
| 218 |
+
memmap_dtype: uint16
|
| 219 |
+
datasets: null
|
| 220 |
+
label_mask_paths: null
|
| 221 |
+
pad_direction: right
|
| 222 |
+
generate_attention_mask: false
|
| 223 |
+
generate_doc_lengths: false
|
| 224 |
+
num_workers: 0
|
| 225 |
+
drop_last: false
|
| 226 |
+
pin_memory: false
|
| 227 |
+
prefetch_factor: null
|
| 228 |
+
persistent_workers: false
|
| 229 |
+
timeout: 0
|
| 230 |
+
seed: null
|
| 231 |
+
instance_filter: null
|
| 232 |
+
device_eval_batch_size: null
|
| 233 |
+
subset_num_batches: null
|
| 234 |
+
- label: winogrande
|
| 235 |
+
type: downstream
|
| 236 |
+
data:
|
| 237 |
+
paths: null
|
| 238 |
+
memmap_dtype: uint16
|
| 239 |
+
datasets: null
|
| 240 |
+
label_mask_paths: null
|
| 241 |
+
pad_direction: right
|
| 242 |
+
generate_attention_mask: false
|
| 243 |
+
generate_doc_lengths: false
|
| 244 |
+
num_workers: 0
|
| 245 |
+
drop_last: false
|
| 246 |
+
pin_memory: false
|
| 247 |
+
prefetch_factor: null
|
| 248 |
+
persistent_workers: false
|
| 249 |
+
timeout: 0
|
| 250 |
+
seed: null
|
| 251 |
+
instance_filter: null
|
| 252 |
+
device_eval_batch_size: null
|
| 253 |
+
subset_num_batches: null
|
| 254 |
+
- label: openbook_qa
|
| 255 |
+
type: downstream
|
| 256 |
+
data:
|
| 257 |
+
paths: null
|
| 258 |
+
memmap_dtype: uint16
|
| 259 |
+
datasets: null
|
| 260 |
+
label_mask_paths: null
|
| 261 |
+
pad_direction: right
|
| 262 |
+
generate_attention_mask: false
|
| 263 |
+
generate_doc_lengths: false
|
| 264 |
+
num_workers: 0
|
| 265 |
+
drop_last: false
|
| 266 |
+
pin_memory: false
|
| 267 |
+
prefetch_factor: null
|
| 268 |
+
persistent_workers: false
|
| 269 |
+
timeout: 0
|
| 270 |
+
seed: null
|
| 271 |
+
instance_filter: null
|
| 272 |
+
device_eval_batch_size: null
|
| 273 |
+
subset_num_batches: null
|
| 274 |
+
- label: boolq
|
| 275 |
+
type: downstream
|
| 276 |
+
data:
|
| 277 |
+
paths: null
|
| 278 |
+
memmap_dtype: uint16
|
| 279 |
+
datasets: null
|
| 280 |
+
label_mask_paths: null
|
| 281 |
+
pad_direction: right
|
| 282 |
+
generate_attention_mask: false
|
| 283 |
+
generate_doc_lengths: false
|
| 284 |
+
num_workers: 0
|
| 285 |
+
drop_last: false
|
| 286 |
+
pin_memory: false
|
| 287 |
+
prefetch_factor: null
|
| 288 |
+
persistent_workers: false
|
| 289 |
+
timeout: 0
|
| 290 |
+
seed: null
|
| 291 |
+
instance_filter: null
|
| 292 |
+
device_eval_batch_size: null
|
| 293 |
+
subset_num_batches: null
|
| 294 |
+
- label: sciq
|
| 295 |
+
type: downstream
|
| 296 |
+
data:
|
| 297 |
+
paths: null
|
| 298 |
+
memmap_dtype: uint16
|
| 299 |
+
datasets: null
|
| 300 |
+
label_mask_paths: null
|
| 301 |
+
pad_direction: right
|
| 302 |
+
generate_attention_mask: false
|
| 303 |
+
generate_doc_lengths: false
|
| 304 |
+
num_workers: 0
|
| 305 |
+
drop_last: false
|
| 306 |
+
pin_memory: false
|
| 307 |
+
prefetch_factor: null
|
| 308 |
+
persistent_workers: false
|
| 309 |
+
timeout: 0
|
| 310 |
+
seed: null
|
| 311 |
+
instance_filter: null
|
| 312 |
+
device_eval_batch_size: null
|
| 313 |
+
subset_num_batches: null
|
| 314 |
+
- label: arc_easy
|
| 315 |
+
type: downstream
|
| 316 |
+
data:
|
| 317 |
+
paths: null
|
| 318 |
+
memmap_dtype: uint16
|
| 319 |
+
datasets: null
|
| 320 |
+
label_mask_paths: null
|
| 321 |
+
pad_direction: right
|
| 322 |
+
generate_attention_mask: false
|
| 323 |
+
generate_doc_lengths: false
|
| 324 |
+
num_workers: 0
|
| 325 |
+
drop_last: false
|
| 326 |
+
pin_memory: false
|
| 327 |
+
prefetch_factor: null
|
| 328 |
+
persistent_workers: false
|
| 329 |
+
timeout: 0
|
| 330 |
+
seed: null
|
| 331 |
+
instance_filter: null
|
| 332 |
+
device_eval_batch_size: null
|
| 333 |
+
subset_num_batches: null
|
| 334 |
+
- label: arc_challenge
|
| 335 |
+
type: downstream
|
| 336 |
+
data:
|
| 337 |
+
paths: null
|
| 338 |
+
memmap_dtype: uint16
|
| 339 |
+
datasets: null
|
| 340 |
+
label_mask_paths: null
|
| 341 |
+
pad_direction: right
|
| 342 |
+
generate_attention_mask: false
|
| 343 |
+
generate_doc_lengths: false
|
| 344 |
+
num_workers: 0
|
| 345 |
+
drop_last: false
|
| 346 |
+
pin_memory: false
|
| 347 |
+
prefetch_factor: null
|
| 348 |
+
persistent_workers: false
|
| 349 |
+
timeout: 0
|
| 350 |
+
seed: null
|
| 351 |
+
instance_filter: null
|
| 352 |
+
device_eval_batch_size: null
|
| 353 |
+
subset_num_batches: null
|
| 354 |
+
- label: copa
|
| 355 |
+
type: downstream
|
| 356 |
+
data:
|
| 357 |
+
paths: null
|
| 358 |
+
memmap_dtype: uint16
|
| 359 |
+
datasets: null
|
| 360 |
+
label_mask_paths: null
|
| 361 |
+
pad_direction: right
|
| 362 |
+
generate_attention_mask: false
|
| 363 |
+
generate_doc_lengths: false
|
| 364 |
+
num_workers: 0
|
| 365 |
+
drop_last: false
|
| 366 |
+
pin_memory: false
|
| 367 |
+
prefetch_factor: null
|
| 368 |
+
persistent_workers: false
|
| 369 |
+
timeout: 0
|
| 370 |
+
seed: null
|
| 371 |
+
instance_filter: null
|
| 372 |
+
device_eval_batch_size: null
|
| 373 |
+
subset_num_batches: null
|
| 374 |
+
- label: commonsense_qa
|
| 375 |
+
type: downstream
|
| 376 |
+
data:
|
| 377 |
+
paths: null
|
| 378 |
+
memmap_dtype: uint16
|
| 379 |
+
datasets: null
|
| 380 |
+
label_mask_paths: null
|
| 381 |
+
pad_direction: right
|
| 382 |
+
generate_attention_mask: false
|
| 383 |
+
generate_doc_lengths: false
|
| 384 |
+
num_workers: 0
|
| 385 |
+
drop_last: false
|
| 386 |
+
pin_memory: false
|
| 387 |
+
prefetch_factor: null
|
| 388 |
+
persistent_workers: false
|
| 389 |
+
timeout: 0
|
| 390 |
+
seed: null
|
| 391 |
+
instance_filter: null
|
| 392 |
+
device_eval_batch_size: null
|
| 393 |
+
subset_num_batches: null
|
| 394 |
+
- label: social_iqa
|
| 395 |
+
type: downstream
|
| 396 |
+
data:
|
| 397 |
+
paths: null
|
| 398 |
+
memmap_dtype: uint16
|
| 399 |
+
datasets: null
|
| 400 |
+
label_mask_paths: null
|
| 401 |
+
pad_direction: right
|
| 402 |
+
generate_attention_mask: false
|
| 403 |
+
generate_doc_lengths: false
|
| 404 |
+
num_workers: 0
|
| 405 |
+
drop_last: false
|
| 406 |
+
pin_memory: false
|
| 407 |
+
prefetch_factor: null
|
| 408 |
+
persistent_workers: false
|
| 409 |
+
timeout: 0
|
| 410 |
+
seed: null
|
| 411 |
+
instance_filter: null
|
| 412 |
+
device_eval_batch_size: null
|
| 413 |
+
subset_num_batches: null
|
| 414 |
+
- label: mmlu_stem_var
|
| 415 |
+
type: downstream
|
| 416 |
+
data:
|
| 417 |
+
paths: null
|
| 418 |
+
memmap_dtype: uint16
|
| 419 |
+
datasets: null
|
| 420 |
+
label_mask_paths: null
|
| 421 |
+
pad_direction: right
|
| 422 |
+
generate_attention_mask: false
|
| 423 |
+
generate_doc_lengths: false
|
| 424 |
+
num_workers: 0
|
| 425 |
+
drop_last: false
|
| 426 |
+
pin_memory: false
|
| 427 |
+
prefetch_factor: null
|
| 428 |
+
persistent_workers: false
|
| 429 |
+
timeout: 0
|
| 430 |
+
seed: null
|
| 431 |
+
instance_filter: null
|
| 432 |
+
device_eval_batch_size: null
|
| 433 |
+
subset_num_batches: null
|
| 434 |
+
- label: mmlu_humanities_var
|
| 435 |
+
type: downstream
|
| 436 |
+
data:
|
| 437 |
+
paths: null
|
| 438 |
+
memmap_dtype: uint16
|
| 439 |
+
datasets: null
|
| 440 |
+
label_mask_paths: null
|
| 441 |
+
pad_direction: right
|
| 442 |
+
generate_attention_mask: false
|
| 443 |
+
generate_doc_lengths: false
|
| 444 |
+
num_workers: 0
|
| 445 |
+
drop_last: false
|
| 446 |
+
pin_memory: false
|
| 447 |
+
prefetch_factor: null
|
| 448 |
+
persistent_workers: false
|
| 449 |
+
timeout: 0
|
| 450 |
+
seed: null
|
| 451 |
+
instance_filter: null
|
| 452 |
+
device_eval_batch_size: null
|
| 453 |
+
subset_num_batches: null
|
| 454 |
+
- label: mmlu_social_sciences_var
|
| 455 |
+
type: downstream
|
| 456 |
+
data:
|
| 457 |
+
paths: null
|
| 458 |
+
memmap_dtype: uint16
|
| 459 |
+
datasets: null
|
| 460 |
+
label_mask_paths: null
|
| 461 |
+
pad_direction: right
|
| 462 |
+
generate_attention_mask: false
|
| 463 |
+
generate_doc_lengths: false
|
| 464 |
+
num_workers: 0
|
| 465 |
+
drop_last: false
|
| 466 |
+
pin_memory: false
|
| 467 |
+
prefetch_factor: null
|
| 468 |
+
persistent_workers: false
|
| 469 |
+
timeout: 0
|
| 470 |
+
seed: null
|
| 471 |
+
instance_filter: null
|
| 472 |
+
device_eval_batch_size: null
|
| 473 |
+
subset_num_batches: null
|
| 474 |
+
- label: mmlu_other_var
|
| 475 |
+
type: downstream
|
| 476 |
+
data:
|
| 477 |
+
paths: null
|
| 478 |
+
memmap_dtype: uint16
|
| 479 |
+
datasets: null
|
| 480 |
+
label_mask_paths: null
|
| 481 |
+
pad_direction: right
|
| 482 |
+
generate_attention_mask: false
|
| 483 |
+
generate_doc_lengths: false
|
| 484 |
+
num_workers: 0
|
| 485 |
+
drop_last: false
|
| 486 |
+
pin_memory: false
|
| 487 |
+
prefetch_factor: null
|
| 488 |
+
persistent_workers: false
|
| 489 |
+
timeout: 0
|
| 490 |
+
seed: null
|
| 491 |
+
instance_filter: null
|
| 492 |
+
device_eval_batch_size: null
|
| 493 |
+
subset_num_batches: null
|
| 494 |
+
- label: mmlu_stem_mc_5shot
|
| 495 |
+
type: downstream
|
| 496 |
+
data:
|
| 497 |
+
paths: null
|
| 498 |
+
memmap_dtype: uint16
|
| 499 |
+
datasets: null
|
| 500 |
+
label_mask_paths: null
|
| 501 |
+
pad_direction: right
|
| 502 |
+
generate_attention_mask: false
|
| 503 |
+
generate_doc_lengths: false
|
| 504 |
+
num_workers: 0
|
| 505 |
+
drop_last: false
|
| 506 |
+
pin_memory: false
|
| 507 |
+
prefetch_factor: null
|
| 508 |
+
persistent_workers: false
|
| 509 |
+
timeout: 0
|
| 510 |
+
seed: null
|
| 511 |
+
instance_filter: null
|
| 512 |
+
device_eval_batch_size: null
|
| 513 |
+
subset_num_batches: null
|
| 514 |
+
- label: mmlu_humanities_mc_5shot
|
| 515 |
+
type: downstream
|
| 516 |
+
data:
|
| 517 |
+
paths: null
|
| 518 |
+
memmap_dtype: uint16
|
| 519 |
+
datasets: null
|
| 520 |
+
label_mask_paths: null
|
| 521 |
+
pad_direction: right
|
| 522 |
+
generate_attention_mask: false
|
| 523 |
+
generate_doc_lengths: false
|
| 524 |
+
num_workers: 0
|
| 525 |
+
drop_last: false
|
| 526 |
+
pin_memory: false
|
| 527 |
+
prefetch_factor: null
|
| 528 |
+
persistent_workers: false
|
| 529 |
+
timeout: 0
|
| 530 |
+
seed: null
|
| 531 |
+
instance_filter: null
|
| 532 |
+
device_eval_batch_size: null
|
| 533 |
+
subset_num_batches: null
|
| 534 |
+
- label: mmlu_social_sciences_mc_5shot
|
| 535 |
+
type: downstream
|
| 536 |
+
data:
|
| 537 |
+
paths: null
|
| 538 |
+
memmap_dtype: uint16
|
| 539 |
+
datasets: null
|
| 540 |
+
label_mask_paths: null
|
| 541 |
+
pad_direction: right
|
| 542 |
+
generate_attention_mask: false
|
| 543 |
+
generate_doc_lengths: false
|
| 544 |
+
num_workers: 0
|
| 545 |
+
drop_last: false
|
| 546 |
+
pin_memory: false
|
| 547 |
+
prefetch_factor: null
|
| 548 |
+
persistent_workers: false
|
| 549 |
+
timeout: 0
|
| 550 |
+
seed: null
|
| 551 |
+
instance_filter: null
|
| 552 |
+
device_eval_batch_size: null
|
| 553 |
+
subset_num_batches: null
|
| 554 |
+
- label: mmlu_other_mc_5shot
|
| 555 |
+
type: downstream
|
| 556 |
+
data:
|
| 557 |
+
paths: null
|
| 558 |
+
memmap_dtype: uint16
|
| 559 |
+
datasets: null
|
| 560 |
+
label_mask_paths: null
|
| 561 |
+
pad_direction: right
|
| 562 |
+
generate_attention_mask: false
|
| 563 |
+
generate_doc_lengths: false
|
| 564 |
+
num_workers: 0
|
| 565 |
+
drop_last: false
|
| 566 |
+
pin_memory: false
|
| 567 |
+
prefetch_factor: null
|
| 568 |
+
persistent_workers: false
|
| 569 |
+
timeout: 0
|
| 570 |
+
seed: null
|
| 571 |
+
instance_filter: null
|
| 572 |
+
device_eval_batch_size: null
|
| 573 |
+
subset_num_batches: null
|
| 574 |
+
- label: mmlu_stem_mc_5shot_test
|
| 575 |
+
type: downstream
|
| 576 |
+
data:
|
| 577 |
+
paths: null
|
| 578 |
+
memmap_dtype: uint16
|
| 579 |
+
datasets: null
|
| 580 |
+
label_mask_paths: null
|
| 581 |
+
pad_direction: right
|
| 582 |
+
generate_attention_mask: false
|
| 583 |
+
generate_doc_lengths: false
|
| 584 |
+
num_workers: 0
|
| 585 |
+
drop_last: false
|
| 586 |
+
pin_memory: false
|
| 587 |
+
prefetch_factor: null
|
| 588 |
+
persistent_workers: false
|
| 589 |
+
timeout: 0
|
| 590 |
+
seed: null
|
| 591 |
+
instance_filter: null
|
| 592 |
+
device_eval_batch_size: null
|
| 593 |
+
subset_num_batches: null
|
| 594 |
+
- label: mmlu_humanities_mc_5shot_test
|
| 595 |
+
type: downstream
|
| 596 |
+
data:
|
| 597 |
+
paths: null
|
| 598 |
+
memmap_dtype: uint16
|
| 599 |
+
datasets: null
|
| 600 |
+
label_mask_paths: null
|
| 601 |
+
pad_direction: right
|
| 602 |
+
generate_attention_mask: false
|
| 603 |
+
generate_doc_lengths: false
|
| 604 |
+
num_workers: 0
|
| 605 |
+
drop_last: false
|
| 606 |
+
pin_memory: false
|
| 607 |
+
prefetch_factor: null
|
| 608 |
+
persistent_workers: false
|
| 609 |
+
timeout: 0
|
| 610 |
+
seed: null
|
| 611 |
+
instance_filter: null
|
| 612 |
+
device_eval_batch_size: null
|
| 613 |
+
subset_num_batches: null
|
| 614 |
+
- label: mmlu_social_sciences_mc_5shot_test
|
| 615 |
+
type: downstream
|
| 616 |
+
data:
|
| 617 |
+
paths: null
|
| 618 |
+
memmap_dtype: uint16
|
| 619 |
+
datasets: null
|
| 620 |
+
label_mask_paths: null
|
| 621 |
+
pad_direction: right
|
| 622 |
+
generate_attention_mask: false
|
| 623 |
+
generate_doc_lengths: false
|
| 624 |
+
num_workers: 0
|
| 625 |
+
drop_last: false
|
| 626 |
+
pin_memory: false
|
| 627 |
+
prefetch_factor: null
|
| 628 |
+
persistent_workers: false
|
| 629 |
+
timeout: 0
|
| 630 |
+
seed: null
|
| 631 |
+
instance_filter: null
|
| 632 |
+
device_eval_batch_size: null
|
| 633 |
+
subset_num_batches: null
|
| 634 |
+
- label: mmlu_other_mc_5shot_test
|
| 635 |
+
type: downstream
|
| 636 |
+
data:
|
| 637 |
+
paths: null
|
| 638 |
+
memmap_dtype: uint16
|
| 639 |
+
datasets: null
|
| 640 |
+
label_mask_paths: null
|
| 641 |
+
pad_direction: right
|
| 642 |
+
generate_attention_mask: false
|
| 643 |
+
generate_doc_lengths: false
|
| 644 |
+
num_workers: 0
|
| 645 |
+
drop_last: false
|
| 646 |
+
pin_memory: false
|
| 647 |
+
prefetch_factor: null
|
| 648 |
+
persistent_workers: false
|
| 649 |
+
timeout: 0
|
| 650 |
+
seed: null
|
| 651 |
+
instance_filter: null
|
| 652 |
+
device_eval_batch_size: null
|
| 653 |
+
subset_num_batches: null
|
| 654 |
+
eval_interval: 2000
|
| 655 |
+
tokenizer:
|
| 656 |
+
identifier: tokenizers/allenai_gpt-neox-olmo-dolma-v1_5.json
|
| 657 |
+
truncate_direction: right
|
| 658 |
+
save_folder: runs/olmoe-7b-29b-2ep
|
| 659 |
+
remote_save_folder: null
|
| 660 |
+
canceled_check_interval: 50
|
| 661 |
+
save_interval: 2000
|
| 662 |
+
save_interval_unsharded: null
|
| 663 |
+
save_interval_ephemeral: null
|
| 664 |
+
save_num_checkpoints_to_keep: 5
|
| 665 |
+
save_num_unsharded_checkpoints_to_keep: -1
|
| 666 |
+
save_overwrite: true
|
| 667 |
+
force_save_unsharded: false
|
| 668 |
+
no_pre_train_checkpoint: true
|
| 669 |
+
load_path: null
|
| 670 |
+
load_path_sharded_checkpointer: null
|
| 671 |
+
try_load_latest_save: false
|
| 672 |
+
reset_optimizer_state: false
|
| 673 |
+
reset_trainer_state: false
|
| 674 |
+
sharded_checkpointer: olmo_core
|
| 675 |
+
new_style_checkpoints: null
|
| 676 |
+
max_duration: 2ep
|
| 677 |
+
global_train_batch_size: 1024
|
| 678 |
+
device_train_batch_size: 256
|
| 679 |
+
device_train_microbatch_size: 2
|
| 680 |
+
device_eval_batch_size: 1
|
| 681 |
+
eval_subset_num_batches: -1
|
| 682 |
+
eval_on_load: false
|
| 683 |
+
device_train_grad_accum: 128
|
| 684 |
+
max_grad_norm: 1.0
|
| 685 |
+
max_grad_norm_ratio: null
|
| 686 |
+
precision: amp_bf16
|
| 687 |
+
wandb:
|
| 688 |
+
project: hiomoe
|
| 689 |
+
entity: bulgakov
|
| 690 |
+
group: 500gb-test
|
| 691 |
+
name: olmoe-7b-29b-2ep
|
| 692 |
+
tags:
|
| 693 |
+
- watching
|
| 694 |
+
log_artifacts: false
|
| 695 |
+
rank_zero_only: true
|
| 696 |
+
log_interval: 1
|
| 697 |
+
speed_monitor:
|
| 698 |
+
window_size: 1
|
| 699 |
+
gpu_flops_available: null
|
| 700 |
+
console_log_interval: 1
|
| 701 |
+
gen1_gc_interval: 1
|
| 702 |
+
compile: null
|
| 703 |
+
distributed_strategy: fsdp
|
| 704 |
+
fsdp:
|
| 705 |
+
use_orig_params: true
|
| 706 |
+
sharding_strategy: FULL_SHARD
|
| 707 |
+
wrapping_strategy: by_block
|
| 708 |
+
precision: mixed
|
| 709 |
+
hybrid_sharding_num_model_replicas: null
|
| 710 |
+
ddp: null
|
| 711 |
+
softmax_auxiliary_loss: false
|
| 712 |
+
auxiliary_loss_multiplier: 0.0001
|
| 713 |
+
time_limit: null
|
| 714 |
+
extra_steps_after_cancel: 10
|
| 715 |
+
early_stopping_factor: null
|
| 716 |
+
save_data_indices: true
|
| 717 |
+
python_profiling: false
|
| 718 |
+
torch_profiling: false
|
| 719 |
+
stop_at: 13964
|
| 720 |
+
stop_after: null
|
| 721 |
+
activation_checkpointing: fine_grained
|
| 722 |
+
fused_loss: null
|
| 723 |
+
hf_datasets_cache_dir: null
|
| 724 |
+
module_outputs_save_steps: null
|
step12000/model/metadata.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"tensors": {"transformer.wte.weight": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 51511808]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[51511808, 103022592]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.ln_f.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 1024]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[1024, 2048]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.ff_out.weight": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 51510784]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[51510784, 103022592]], "local_shape": null, "global_offset": null}}}}}
|
step12000/optim/metadata.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
step12000/train/rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5d87bab02ce17c12a45a78b586cbb9120d29e03127306bfc91bd6dc44bbcc6e1
|
| 3 |
+
size 15061
|
step12000/train/rank1.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5bd04f1d6abcbb4002833ee6e46eef7fac65e44d625ab7b87eca22d82b242940
|
| 3 |
+
size 15061
|
step12000/train/rank2.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9406840828e8ffe5ec25a17d0b8c2a42ba439ee3cd08f7aa2e301d12a38f666f
|
| 3 |
+
size 15061
|
step12000/train/rank3.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:cf4f7989b941e7695d4d8989b9997e97791ca12658e880f8916c49d1c51bab90
|
| 3 |
+
size 15061
|
step13954/config.yaml
ADDED
|
@@ -0,0 +1,724 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
run_name: olmoe-7b-29b-2ep
|
| 2 |
+
seed: 6198
|
| 3 |
+
epoch: null
|
| 4 |
+
dry_run: false
|
| 5 |
+
model:
|
| 6 |
+
d_model: 2048
|
| 7 |
+
n_heads: 16
|
| 8 |
+
n_kv_heads: null
|
| 9 |
+
clip_qkv: null
|
| 10 |
+
n_layers: 16
|
| 11 |
+
mlp_ratio: 1
|
| 12 |
+
mlp_hidden_size: null
|
| 13 |
+
activation_type: swiglu
|
| 14 |
+
block_type: moe
|
| 15 |
+
block_group_size: 1
|
| 16 |
+
alibi: false
|
| 17 |
+
alibi_bias_max: 8.0
|
| 18 |
+
rope: true
|
| 19 |
+
rope_full_precision: true
|
| 20 |
+
rope_theta: 10000
|
| 21 |
+
flash_attention: true
|
| 22 |
+
attention_dropout: 0.0
|
| 23 |
+
multi_query_attention: null
|
| 24 |
+
attention_layer_norm: true
|
| 25 |
+
residual_dropout: 0.0
|
| 26 |
+
embedding_dropout: 0.0
|
| 27 |
+
embedding_layer_norm: false
|
| 28 |
+
layer_norm_type: rms
|
| 29 |
+
layer_norm_with_affine: true
|
| 30 |
+
layer_norm_eps: 1.0e-05
|
| 31 |
+
attention_layer_norm_with_affine: true
|
| 32 |
+
max_sequence_length: 4096
|
| 33 |
+
include_bias: false
|
| 34 |
+
bias_for_layer_norm: false
|
| 35 |
+
scale_logits: false
|
| 36 |
+
vocab_size: 50280
|
| 37 |
+
embedding_size: 50304
|
| 38 |
+
weight_tying: false
|
| 39 |
+
eos_token_id: 0
|
| 40 |
+
pad_token_id: 1
|
| 41 |
+
init_device: meta
|
| 42 |
+
init_fn: normal
|
| 43 |
+
init_std: 0.02
|
| 44 |
+
init_cutoff_factor: 3.0
|
| 45 |
+
precision: amp_bf16
|
| 46 |
+
moe_num_experts: 64
|
| 47 |
+
moe_top_k: 8
|
| 48 |
+
moe_mlp_impl: sparse
|
| 49 |
+
moe_log_expert_assignment: false
|
| 50 |
+
moe_shared_expert: false
|
| 51 |
+
moe_lbl_in_fp32: false
|
| 52 |
+
moe_loss_weight: 0.01
|
| 53 |
+
moe_zloss_weight: 0.001
|
| 54 |
+
moe_dropless: false
|
| 55 |
+
moe_capacity_factor: 1.25
|
| 56 |
+
himoe_enabled: false
|
| 57 |
+
himoe_n_groups: 4
|
| 58 |
+
himoe_top_k_per_group: 2
|
| 59 |
+
himoe_lambda1: 0.01
|
| 60 |
+
himoe_lambda2: 0.1
|
| 61 |
+
scale_emb_init: false
|
| 62 |
+
emb_init_std: null
|
| 63 |
+
norm_after: false
|
| 64 |
+
optimizer:
|
| 65 |
+
name: adamw
|
| 66 |
+
learning_rate: 0.0004
|
| 67 |
+
weight_decay: 0.1
|
| 68 |
+
betas:
|
| 69 |
+
- 0.9
|
| 70 |
+
- 0.95
|
| 71 |
+
eps: 1.0e-08
|
| 72 |
+
no_decay_norm_and_bias: null
|
| 73 |
+
selective_updates: false
|
| 74 |
+
decay_norm_and_bias: true
|
| 75 |
+
decay_embeddings: true
|
| 76 |
+
metrics_log_interval: 10
|
| 77 |
+
record_update_metrics: false
|
| 78 |
+
scheduler:
|
| 79 |
+
name: cosine_with_warmup
|
| 80 |
+
units: tokens
|
| 81 |
+
t_warmup: 1405296000
|
| 82 |
+
t_max: 58540000000
|
| 83 |
+
alpha_f: 0.1
|
| 84 |
+
grad_clip_warmup_steps: null
|
| 85 |
+
grad_clip_warmup_factor: null
|
| 86 |
+
warmup_min_lr: null
|
| 87 |
+
data:
|
| 88 |
+
paths:
|
| 89 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy
|
| 90 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy
|
| 91 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy
|
| 92 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy
|
| 93 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy
|
| 94 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy
|
| 95 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy
|
| 96 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy
|
| 97 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy
|
| 98 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy
|
| 99 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy
|
| 100 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy
|
| 101 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy
|
| 102 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-13-00000.npy
|
| 103 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-14-00000.npy
|
| 104 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-15-00000.npy
|
| 105 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-000-00000.npy
|
| 106 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-001-00000.npy
|
| 107 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-002-00000.npy
|
| 108 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-003-00000.npy
|
| 109 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-004-00000.npy
|
| 110 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-005-00000.npy
|
| 111 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-006-00000.npy
|
| 112 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-007-00000.npy
|
| 113 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-008-00000.npy
|
| 114 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-009-00000.npy
|
| 115 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-010-00000.npy
|
| 116 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-011-00000.npy
|
| 117 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-012-00000.npy
|
| 118 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-013-00000.npy
|
| 119 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy
|
| 120 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy
|
| 121 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy
|
| 122 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy
|
| 123 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy
|
| 124 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy
|
| 125 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy
|
| 126 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy
|
| 127 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy
|
| 128 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy
|
| 129 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy
|
| 130 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy
|
| 131 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy
|
| 132 |
+
memmap_dtype: uint16
|
| 133 |
+
datasets: null
|
| 134 |
+
label_mask_paths: null
|
| 135 |
+
pad_direction: right
|
| 136 |
+
generate_attention_mask: false
|
| 137 |
+
generate_doc_lengths: false
|
| 138 |
+
num_workers: 8
|
| 139 |
+
drop_last: true
|
| 140 |
+
pin_memory: true
|
| 141 |
+
prefetch_factor: 8
|
| 142 |
+
persistent_workers: true
|
| 143 |
+
timeout: 0
|
| 144 |
+
seed: null
|
| 145 |
+
instance_filter:
|
| 146 |
+
repetition_max_period: 13
|
| 147 |
+
repetition_min_period: 1
|
| 148 |
+
repetition_max_count: 32
|
| 149 |
+
restore_dataloader: true
|
| 150 |
+
fast_forward_batches: null
|
| 151 |
+
evaluators:
|
| 152 |
+
- label: all-small-ppl-validation
|
| 153 |
+
type: lm
|
| 154 |
+
data:
|
| 155 |
+
paths: null
|
| 156 |
+
memmap_dtype: uint16
|
| 157 |
+
datasets:
|
| 158 |
+
c4_en-validation:
|
| 159 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/c4_en/val/part-0-00000.npy
|
| 160 |
+
dolma_books-validation:
|
| 161 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_books/val/part-0-00000.npy
|
| 162 |
+
dolma_common-crawl-validation:
|
| 163 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_common-crawl/val/part-0-00000.npy
|
| 164 |
+
dolma_pes2o-validation:
|
| 165 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_pes2o/val/part-0-00000.npy
|
| 166 |
+
dolma_reddit-validation:
|
| 167 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_reddit/val/part-0-00000.npy
|
| 168 |
+
dolma_stack-validation:
|
| 169 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_stack/val/part-0-00000.npy
|
| 170 |
+
dolma_wiki-validation:
|
| 171 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_wiki/val/part-0-00000.npy
|
| 172 |
+
ice-validation:
|
| 173 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/ice/val/part-0-00000.npy
|
| 174 |
+
m2d2_s2orc-validation:
|
| 175 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/m2d2_s2orc/val/part-0-00000.npy
|
| 176 |
+
pile-validation:
|
| 177 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/pile/val/part-0-00000.npy
|
| 178 |
+
wikitext_103-validation:
|
| 179 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/wikitext_103/val/part-0-00000.npy
|
| 180 |
+
label_mask_paths: null
|
| 181 |
+
pad_direction: right
|
| 182 |
+
generate_attention_mask: false
|
| 183 |
+
generate_doc_lengths: false
|
| 184 |
+
num_workers: 0
|
| 185 |
+
drop_last: true
|
| 186 |
+
pin_memory: false
|
| 187 |
+
prefetch_factor: null
|
| 188 |
+
persistent_workers: false
|
| 189 |
+
timeout: 0
|
| 190 |
+
seed: null
|
| 191 |
+
instance_filter: null
|
| 192 |
+
device_eval_batch_size: null
|
| 193 |
+
subset_num_batches: null
|
| 194 |
+
- label: piqa
|
| 195 |
+
type: downstream
|
| 196 |
+
data:
|
| 197 |
+
paths: null
|
| 198 |
+
memmap_dtype: uint16
|
| 199 |
+
datasets: null
|
| 200 |
+
label_mask_paths: null
|
| 201 |
+
pad_direction: right
|
| 202 |
+
generate_attention_mask: false
|
| 203 |
+
generate_doc_lengths: false
|
| 204 |
+
num_workers: 0
|
| 205 |
+
drop_last: false
|
| 206 |
+
pin_memory: false
|
| 207 |
+
prefetch_factor: null
|
| 208 |
+
persistent_workers: false
|
| 209 |
+
timeout: 0
|
| 210 |
+
seed: null
|
| 211 |
+
instance_filter: null
|
| 212 |
+
device_eval_batch_size: null
|
| 213 |
+
subset_num_batches: null
|
| 214 |
+
- label: hellaswag
|
| 215 |
+
type: downstream
|
| 216 |
+
data:
|
| 217 |
+
paths: null
|
| 218 |
+
memmap_dtype: uint16
|
| 219 |
+
datasets: null
|
| 220 |
+
label_mask_paths: null
|
| 221 |
+
pad_direction: right
|
| 222 |
+
generate_attention_mask: false
|
| 223 |
+
generate_doc_lengths: false
|
| 224 |
+
num_workers: 0
|
| 225 |
+
drop_last: false
|
| 226 |
+
pin_memory: false
|
| 227 |
+
prefetch_factor: null
|
| 228 |
+
persistent_workers: false
|
| 229 |
+
timeout: 0
|
| 230 |
+
seed: null
|
| 231 |
+
instance_filter: null
|
| 232 |
+
device_eval_batch_size: null
|
| 233 |
+
subset_num_batches: null
|
| 234 |
+
- label: winogrande
|
| 235 |
+
type: downstream
|
| 236 |
+
data:
|
| 237 |
+
paths: null
|
| 238 |
+
memmap_dtype: uint16
|
| 239 |
+
datasets: null
|
| 240 |
+
label_mask_paths: null
|
| 241 |
+
pad_direction: right
|
| 242 |
+
generate_attention_mask: false
|
| 243 |
+
generate_doc_lengths: false
|
| 244 |
+
num_workers: 0
|
| 245 |
+
drop_last: false
|
| 246 |
+
pin_memory: false
|
| 247 |
+
prefetch_factor: null
|
| 248 |
+
persistent_workers: false
|
| 249 |
+
timeout: 0
|
| 250 |
+
seed: null
|
| 251 |
+
instance_filter: null
|
| 252 |
+
device_eval_batch_size: null
|
| 253 |
+
subset_num_batches: null
|
| 254 |
+
- label: openbook_qa
|
| 255 |
+
type: downstream
|
| 256 |
+
data:
|
| 257 |
+
paths: null
|
| 258 |
+
memmap_dtype: uint16
|
| 259 |
+
datasets: null
|
| 260 |
+
label_mask_paths: null
|
| 261 |
+
pad_direction: right
|
| 262 |
+
generate_attention_mask: false
|
| 263 |
+
generate_doc_lengths: false
|
| 264 |
+
num_workers: 0
|
| 265 |
+
drop_last: false
|
| 266 |
+
pin_memory: false
|
| 267 |
+
prefetch_factor: null
|
| 268 |
+
persistent_workers: false
|
| 269 |
+
timeout: 0
|
| 270 |
+
seed: null
|
| 271 |
+
instance_filter: null
|
| 272 |
+
device_eval_batch_size: null
|
| 273 |
+
subset_num_batches: null
|
| 274 |
+
- label: boolq
|
| 275 |
+
type: downstream
|
| 276 |
+
data:
|
| 277 |
+
paths: null
|
| 278 |
+
memmap_dtype: uint16
|
| 279 |
+
datasets: null
|
| 280 |
+
label_mask_paths: null
|
| 281 |
+
pad_direction: right
|
| 282 |
+
generate_attention_mask: false
|
| 283 |
+
generate_doc_lengths: false
|
| 284 |
+
num_workers: 0
|
| 285 |
+
drop_last: false
|
| 286 |
+
pin_memory: false
|
| 287 |
+
prefetch_factor: null
|
| 288 |
+
persistent_workers: false
|
| 289 |
+
timeout: 0
|
| 290 |
+
seed: null
|
| 291 |
+
instance_filter: null
|
| 292 |
+
device_eval_batch_size: null
|
| 293 |
+
subset_num_batches: null
|
| 294 |
+
- label: sciq
|
| 295 |
+
type: downstream
|
| 296 |
+
data:
|
| 297 |
+
paths: null
|
| 298 |
+
memmap_dtype: uint16
|
| 299 |
+
datasets: null
|
| 300 |
+
label_mask_paths: null
|
| 301 |
+
pad_direction: right
|
| 302 |
+
generate_attention_mask: false
|
| 303 |
+
generate_doc_lengths: false
|
| 304 |
+
num_workers: 0
|
| 305 |
+
drop_last: false
|
| 306 |
+
pin_memory: false
|
| 307 |
+
prefetch_factor: null
|
| 308 |
+
persistent_workers: false
|
| 309 |
+
timeout: 0
|
| 310 |
+
seed: null
|
| 311 |
+
instance_filter: null
|
| 312 |
+
device_eval_batch_size: null
|
| 313 |
+
subset_num_batches: null
|
| 314 |
+
- label: arc_easy
|
| 315 |
+
type: downstream
|
| 316 |
+
data:
|
| 317 |
+
paths: null
|
| 318 |
+
memmap_dtype: uint16
|
| 319 |
+
datasets: null
|
| 320 |
+
label_mask_paths: null
|
| 321 |
+
pad_direction: right
|
| 322 |
+
generate_attention_mask: false
|
| 323 |
+
generate_doc_lengths: false
|
| 324 |
+
num_workers: 0
|
| 325 |
+
drop_last: false
|
| 326 |
+
pin_memory: false
|
| 327 |
+
prefetch_factor: null
|
| 328 |
+
persistent_workers: false
|
| 329 |
+
timeout: 0
|
| 330 |
+
seed: null
|
| 331 |
+
instance_filter: null
|
| 332 |
+
device_eval_batch_size: null
|
| 333 |
+
subset_num_batches: null
|
| 334 |
+
- label: arc_challenge
|
| 335 |
+
type: downstream
|
| 336 |
+
data:
|
| 337 |
+
paths: null
|
| 338 |
+
memmap_dtype: uint16
|
| 339 |
+
datasets: null
|
| 340 |
+
label_mask_paths: null
|
| 341 |
+
pad_direction: right
|
| 342 |
+
generate_attention_mask: false
|
| 343 |
+
generate_doc_lengths: false
|
| 344 |
+
num_workers: 0
|
| 345 |
+
drop_last: false
|
| 346 |
+
pin_memory: false
|
| 347 |
+
prefetch_factor: null
|
| 348 |
+
persistent_workers: false
|
| 349 |
+
timeout: 0
|
| 350 |
+
seed: null
|
| 351 |
+
instance_filter: null
|
| 352 |
+
device_eval_batch_size: null
|
| 353 |
+
subset_num_batches: null
|
| 354 |
+
- label: copa
|
| 355 |
+
type: downstream
|
| 356 |
+
data:
|
| 357 |
+
paths: null
|
| 358 |
+
memmap_dtype: uint16
|
| 359 |
+
datasets: null
|
| 360 |
+
label_mask_paths: null
|
| 361 |
+
pad_direction: right
|
| 362 |
+
generate_attention_mask: false
|
| 363 |
+
generate_doc_lengths: false
|
| 364 |
+
num_workers: 0
|
| 365 |
+
drop_last: false
|
| 366 |
+
pin_memory: false
|
| 367 |
+
prefetch_factor: null
|
| 368 |
+
persistent_workers: false
|
| 369 |
+
timeout: 0
|
| 370 |
+
seed: null
|
| 371 |
+
instance_filter: null
|
| 372 |
+
device_eval_batch_size: null
|
| 373 |
+
subset_num_batches: null
|
| 374 |
+
- label: commonsense_qa
|
| 375 |
+
type: downstream
|
| 376 |
+
data:
|
| 377 |
+
paths: null
|
| 378 |
+
memmap_dtype: uint16
|
| 379 |
+
datasets: null
|
| 380 |
+
label_mask_paths: null
|
| 381 |
+
pad_direction: right
|
| 382 |
+
generate_attention_mask: false
|
| 383 |
+
generate_doc_lengths: false
|
| 384 |
+
num_workers: 0
|
| 385 |
+
drop_last: false
|
| 386 |
+
pin_memory: false
|
| 387 |
+
prefetch_factor: null
|
| 388 |
+
persistent_workers: false
|
| 389 |
+
timeout: 0
|
| 390 |
+
seed: null
|
| 391 |
+
instance_filter: null
|
| 392 |
+
device_eval_batch_size: null
|
| 393 |
+
subset_num_batches: null
|
| 394 |
+
- label: social_iqa
|
| 395 |
+
type: downstream
|
| 396 |
+
data:
|
| 397 |
+
paths: null
|
| 398 |
+
memmap_dtype: uint16
|
| 399 |
+
datasets: null
|
| 400 |
+
label_mask_paths: null
|
| 401 |
+
pad_direction: right
|
| 402 |
+
generate_attention_mask: false
|
| 403 |
+
generate_doc_lengths: false
|
| 404 |
+
num_workers: 0
|
| 405 |
+
drop_last: false
|
| 406 |
+
pin_memory: false
|
| 407 |
+
prefetch_factor: null
|
| 408 |
+
persistent_workers: false
|
| 409 |
+
timeout: 0
|
| 410 |
+
seed: null
|
| 411 |
+
instance_filter: null
|
| 412 |
+
device_eval_batch_size: null
|
| 413 |
+
subset_num_batches: null
|
| 414 |
+
- label: mmlu_stem_var
|
| 415 |
+
type: downstream
|
| 416 |
+
data:
|
| 417 |
+
paths: null
|
| 418 |
+
memmap_dtype: uint16
|
| 419 |
+
datasets: null
|
| 420 |
+
label_mask_paths: null
|
| 421 |
+
pad_direction: right
|
| 422 |
+
generate_attention_mask: false
|
| 423 |
+
generate_doc_lengths: false
|
| 424 |
+
num_workers: 0
|
| 425 |
+
drop_last: false
|
| 426 |
+
pin_memory: false
|
| 427 |
+
prefetch_factor: null
|
| 428 |
+
persistent_workers: false
|
| 429 |
+
timeout: 0
|
| 430 |
+
seed: null
|
| 431 |
+
instance_filter: null
|
| 432 |
+
device_eval_batch_size: null
|
| 433 |
+
subset_num_batches: null
|
| 434 |
+
- label: mmlu_humanities_var
|
| 435 |
+
type: downstream
|
| 436 |
+
data:
|
| 437 |
+
paths: null
|
| 438 |
+
memmap_dtype: uint16
|
| 439 |
+
datasets: null
|
| 440 |
+
label_mask_paths: null
|
| 441 |
+
pad_direction: right
|
| 442 |
+
generate_attention_mask: false
|
| 443 |
+
generate_doc_lengths: false
|
| 444 |
+
num_workers: 0
|
| 445 |
+
drop_last: false
|
| 446 |
+
pin_memory: false
|
| 447 |
+
prefetch_factor: null
|
| 448 |
+
persistent_workers: false
|
| 449 |
+
timeout: 0
|
| 450 |
+
seed: null
|
| 451 |
+
instance_filter: null
|
| 452 |
+
device_eval_batch_size: null
|
| 453 |
+
subset_num_batches: null
|
| 454 |
+
- label: mmlu_social_sciences_var
|
| 455 |
+
type: downstream
|
| 456 |
+
data:
|
| 457 |
+
paths: null
|
| 458 |
+
memmap_dtype: uint16
|
| 459 |
+
datasets: null
|
| 460 |
+
label_mask_paths: null
|
| 461 |
+
pad_direction: right
|
| 462 |
+
generate_attention_mask: false
|
| 463 |
+
generate_doc_lengths: false
|
| 464 |
+
num_workers: 0
|
| 465 |
+
drop_last: false
|
| 466 |
+
pin_memory: false
|
| 467 |
+
prefetch_factor: null
|
| 468 |
+
persistent_workers: false
|
| 469 |
+
timeout: 0
|
| 470 |
+
seed: null
|
| 471 |
+
instance_filter: null
|
| 472 |
+
device_eval_batch_size: null
|
| 473 |
+
subset_num_batches: null
|
| 474 |
+
- label: mmlu_other_var
|
| 475 |
+
type: downstream
|
| 476 |
+
data:
|
| 477 |
+
paths: null
|
| 478 |
+
memmap_dtype: uint16
|
| 479 |
+
datasets: null
|
| 480 |
+
label_mask_paths: null
|
| 481 |
+
pad_direction: right
|
| 482 |
+
generate_attention_mask: false
|
| 483 |
+
generate_doc_lengths: false
|
| 484 |
+
num_workers: 0
|
| 485 |
+
drop_last: false
|
| 486 |
+
pin_memory: false
|
| 487 |
+
prefetch_factor: null
|
| 488 |
+
persistent_workers: false
|
| 489 |
+
timeout: 0
|
| 490 |
+
seed: null
|
| 491 |
+
instance_filter: null
|
| 492 |
+
device_eval_batch_size: null
|
| 493 |
+
subset_num_batches: null
|
| 494 |
+
- label: mmlu_stem_mc_5shot
|
| 495 |
+
type: downstream
|
| 496 |
+
data:
|
| 497 |
+
paths: null
|
| 498 |
+
memmap_dtype: uint16
|
| 499 |
+
datasets: null
|
| 500 |
+
label_mask_paths: null
|
| 501 |
+
pad_direction: right
|
| 502 |
+
generate_attention_mask: false
|
| 503 |
+
generate_doc_lengths: false
|
| 504 |
+
num_workers: 0
|
| 505 |
+
drop_last: false
|
| 506 |
+
pin_memory: false
|
| 507 |
+
prefetch_factor: null
|
| 508 |
+
persistent_workers: false
|
| 509 |
+
timeout: 0
|
| 510 |
+
seed: null
|
| 511 |
+
instance_filter: null
|
| 512 |
+
device_eval_batch_size: null
|
| 513 |
+
subset_num_batches: null
|
| 514 |
+
- label: mmlu_humanities_mc_5shot
|
| 515 |
+
type: downstream
|
| 516 |
+
data:
|
| 517 |
+
paths: null
|
| 518 |
+
memmap_dtype: uint16
|
| 519 |
+
datasets: null
|
| 520 |
+
label_mask_paths: null
|
| 521 |
+
pad_direction: right
|
| 522 |
+
generate_attention_mask: false
|
| 523 |
+
generate_doc_lengths: false
|
| 524 |
+
num_workers: 0
|
| 525 |
+
drop_last: false
|
| 526 |
+
pin_memory: false
|
| 527 |
+
prefetch_factor: null
|
| 528 |
+
persistent_workers: false
|
| 529 |
+
timeout: 0
|
| 530 |
+
seed: null
|
| 531 |
+
instance_filter: null
|
| 532 |
+
device_eval_batch_size: null
|
| 533 |
+
subset_num_batches: null
|
| 534 |
+
- label: mmlu_social_sciences_mc_5shot
|
| 535 |
+
type: downstream
|
| 536 |
+
data:
|
| 537 |
+
paths: null
|
| 538 |
+
memmap_dtype: uint16
|
| 539 |
+
datasets: null
|
| 540 |
+
label_mask_paths: null
|
| 541 |
+
pad_direction: right
|
| 542 |
+
generate_attention_mask: false
|
| 543 |
+
generate_doc_lengths: false
|
| 544 |
+
num_workers: 0
|
| 545 |
+
drop_last: false
|
| 546 |
+
pin_memory: false
|
| 547 |
+
prefetch_factor: null
|
| 548 |
+
persistent_workers: false
|
| 549 |
+
timeout: 0
|
| 550 |
+
seed: null
|
| 551 |
+
instance_filter: null
|
| 552 |
+
device_eval_batch_size: null
|
| 553 |
+
subset_num_batches: null
|
| 554 |
+
- label: mmlu_other_mc_5shot
|
| 555 |
+
type: downstream
|
| 556 |
+
data:
|
| 557 |
+
paths: null
|
| 558 |
+
memmap_dtype: uint16
|
| 559 |
+
datasets: null
|
| 560 |
+
label_mask_paths: null
|
| 561 |
+
pad_direction: right
|
| 562 |
+
generate_attention_mask: false
|
| 563 |
+
generate_doc_lengths: false
|
| 564 |
+
num_workers: 0
|
| 565 |
+
drop_last: false
|
| 566 |
+
pin_memory: false
|
| 567 |
+
prefetch_factor: null
|
| 568 |
+
persistent_workers: false
|
| 569 |
+
timeout: 0
|
| 570 |
+
seed: null
|
| 571 |
+
instance_filter: null
|
| 572 |
+
device_eval_batch_size: null
|
| 573 |
+
subset_num_batches: null
|
| 574 |
+
- label: mmlu_stem_mc_5shot_test
|
| 575 |
+
type: downstream
|
| 576 |
+
data:
|
| 577 |
+
paths: null
|
| 578 |
+
memmap_dtype: uint16
|
| 579 |
+
datasets: null
|
| 580 |
+
label_mask_paths: null
|
| 581 |
+
pad_direction: right
|
| 582 |
+
generate_attention_mask: false
|
| 583 |
+
generate_doc_lengths: false
|
| 584 |
+
num_workers: 0
|
| 585 |
+
drop_last: false
|
| 586 |
+
pin_memory: false
|
| 587 |
+
prefetch_factor: null
|
| 588 |
+
persistent_workers: false
|
| 589 |
+
timeout: 0
|
| 590 |
+
seed: null
|
| 591 |
+
instance_filter: null
|
| 592 |
+
device_eval_batch_size: null
|
| 593 |
+
subset_num_batches: null
|
| 594 |
+
- label: mmlu_humanities_mc_5shot_test
|
| 595 |
+
type: downstream
|
| 596 |
+
data:
|
| 597 |
+
paths: null
|
| 598 |
+
memmap_dtype: uint16
|
| 599 |
+
datasets: null
|
| 600 |
+
label_mask_paths: null
|
| 601 |
+
pad_direction: right
|
| 602 |
+
generate_attention_mask: false
|
| 603 |
+
generate_doc_lengths: false
|
| 604 |
+
num_workers: 0
|
| 605 |
+
drop_last: false
|
| 606 |
+
pin_memory: false
|
| 607 |
+
prefetch_factor: null
|
| 608 |
+
persistent_workers: false
|
| 609 |
+
timeout: 0
|
| 610 |
+
seed: null
|
| 611 |
+
instance_filter: null
|
| 612 |
+
device_eval_batch_size: null
|
| 613 |
+
subset_num_batches: null
|
| 614 |
+
- label: mmlu_social_sciences_mc_5shot_test
|
| 615 |
+
type: downstream
|
| 616 |
+
data:
|
| 617 |
+
paths: null
|
| 618 |
+
memmap_dtype: uint16
|
| 619 |
+
datasets: null
|
| 620 |
+
label_mask_paths: null
|
| 621 |
+
pad_direction: right
|
| 622 |
+
generate_attention_mask: false
|
| 623 |
+
generate_doc_lengths: false
|
| 624 |
+
num_workers: 0
|
| 625 |
+
drop_last: false
|
| 626 |
+
pin_memory: false
|
| 627 |
+
prefetch_factor: null
|
| 628 |
+
persistent_workers: false
|
| 629 |
+
timeout: 0
|
| 630 |
+
seed: null
|
| 631 |
+
instance_filter: null
|
| 632 |
+
device_eval_batch_size: null
|
| 633 |
+
subset_num_batches: null
|
| 634 |
+
- label: mmlu_other_mc_5shot_test
|
| 635 |
+
type: downstream
|
| 636 |
+
data:
|
| 637 |
+
paths: null
|
| 638 |
+
memmap_dtype: uint16
|
| 639 |
+
datasets: null
|
| 640 |
+
label_mask_paths: null
|
| 641 |
+
pad_direction: right
|
| 642 |
+
generate_attention_mask: false
|
| 643 |
+
generate_doc_lengths: false
|
| 644 |
+
num_workers: 0
|
| 645 |
+
drop_last: false
|
| 646 |
+
pin_memory: false
|
| 647 |
+
prefetch_factor: null
|
| 648 |
+
persistent_workers: false
|
| 649 |
+
timeout: 0
|
| 650 |
+
seed: null
|
| 651 |
+
instance_filter: null
|
| 652 |
+
device_eval_batch_size: null
|
| 653 |
+
subset_num_batches: null
|
| 654 |
+
eval_interval: 2000
|
| 655 |
+
tokenizer:
|
| 656 |
+
identifier: tokenizers/allenai_gpt-neox-olmo-dolma-v1_5.json
|
| 657 |
+
truncate_direction: right
|
| 658 |
+
save_folder: runs/olmoe-7b-29b-2ep
|
| 659 |
+
remote_save_folder: null
|
| 660 |
+
canceled_check_interval: 50
|
| 661 |
+
save_interval: 2000
|
| 662 |
+
save_interval_unsharded: null
|
| 663 |
+
save_interval_ephemeral: null
|
| 664 |
+
save_num_checkpoints_to_keep: 5
|
| 665 |
+
save_num_unsharded_checkpoints_to_keep: -1
|
| 666 |
+
save_overwrite: true
|
| 667 |
+
force_save_unsharded: false
|
| 668 |
+
no_pre_train_checkpoint: true
|
| 669 |
+
load_path: null
|
| 670 |
+
load_path_sharded_checkpointer: null
|
| 671 |
+
try_load_latest_save: false
|
| 672 |
+
reset_optimizer_state: false
|
| 673 |
+
reset_trainer_state: false
|
| 674 |
+
sharded_checkpointer: olmo_core
|
| 675 |
+
new_style_checkpoints: null
|
| 676 |
+
max_duration: 2ep
|
| 677 |
+
global_train_batch_size: 1024
|
| 678 |
+
device_train_batch_size: 256
|
| 679 |
+
device_train_microbatch_size: 2
|
| 680 |
+
device_eval_batch_size: 1
|
| 681 |
+
eval_subset_num_batches: -1
|
| 682 |
+
eval_on_load: false
|
| 683 |
+
device_train_grad_accum: 128
|
| 684 |
+
max_grad_norm: 1.0
|
| 685 |
+
max_grad_norm_ratio: null
|
| 686 |
+
precision: amp_bf16
|
| 687 |
+
wandb:
|
| 688 |
+
project: hiomoe
|
| 689 |
+
entity: bulgakov
|
| 690 |
+
group: 500gb-test
|
| 691 |
+
name: olmoe-7b-29b-2ep
|
| 692 |
+
tags:
|
| 693 |
+
- watching
|
| 694 |
+
log_artifacts: false
|
| 695 |
+
rank_zero_only: true
|
| 696 |
+
log_interval: 1
|
| 697 |
+
speed_monitor:
|
| 698 |
+
window_size: 1
|
| 699 |
+
gpu_flops_available: null
|
| 700 |
+
console_log_interval: 1
|
| 701 |
+
gen1_gc_interval: 1
|
| 702 |
+
compile: null
|
| 703 |
+
distributed_strategy: fsdp
|
| 704 |
+
fsdp:
|
| 705 |
+
use_orig_params: true
|
| 706 |
+
sharding_strategy: FULL_SHARD
|
| 707 |
+
wrapping_strategy: by_block
|
| 708 |
+
precision: mixed
|
| 709 |
+
hybrid_sharding_num_model_replicas: null
|
| 710 |
+
ddp: null
|
| 711 |
+
softmax_auxiliary_loss: false
|
| 712 |
+
auxiliary_loss_multiplier: 0.0001
|
| 713 |
+
time_limit: null
|
| 714 |
+
extra_steps_after_cancel: 10
|
| 715 |
+
early_stopping_factor: null
|
| 716 |
+
save_data_indices: true
|
| 717 |
+
python_profiling: false
|
| 718 |
+
torch_profiling: false
|
| 719 |
+
stop_at: 13964
|
| 720 |
+
stop_after: null
|
| 721 |
+
activation_checkpointing: fine_grained
|
| 722 |
+
fused_loss: null
|
| 723 |
+
hf_datasets_cache_dir: null
|
| 724 |
+
module_outputs_save_steps: null
|
step13954/model/metadata.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"tensors": {"transformer.wte.weight": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 51511808]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[51511808, 103022592]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.ln_f.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 1024]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[1024, 2048]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.ff_out.weight": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 51510784]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[51510784, 103022592]], "local_shape": null, "global_offset": null}}}}}
|
step13954/optim/metadata.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
step13954/train/rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a0132cb0d342bbcb6da79c45648ac7a87b83ba054aed6192faf46190f9d03b41
|
| 3 |
+
size 15061
|
step13954/train/rank1.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9070989c555d4e0195bcbb03e97b71296fadff0ed280c77758bf7eff7fe100d7
|
| 3 |
+
size 15061
|
step13954/train/rank2.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d52aaa729bcdd1ca6b2b3cfd700c1eb99f2b7984f5db75ecac2c15715b18f0e6
|
| 3 |
+
size 15061
|
step13954/train/rank3.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0be30cfbd412cee2d35d4e71acd9e5cd11df24c02b5a81b30d97cdfc6ce757e4
|
| 3 |
+
size 15061
|
step6000/config.yaml
ADDED
|
@@ -0,0 +1,724 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
run_name: olmoe-7b-29b-2ep
|
| 2 |
+
seed: 6198
|
| 3 |
+
epoch: null
|
| 4 |
+
dry_run: false
|
| 5 |
+
model:
|
| 6 |
+
d_model: 2048
|
| 7 |
+
n_heads: 16
|
| 8 |
+
n_kv_heads: null
|
| 9 |
+
clip_qkv: null
|
| 10 |
+
n_layers: 16
|
| 11 |
+
mlp_ratio: 1
|
| 12 |
+
mlp_hidden_size: null
|
| 13 |
+
activation_type: swiglu
|
| 14 |
+
block_type: moe
|
| 15 |
+
block_group_size: 1
|
| 16 |
+
alibi: false
|
| 17 |
+
alibi_bias_max: 8.0
|
| 18 |
+
rope: true
|
| 19 |
+
rope_full_precision: true
|
| 20 |
+
rope_theta: 10000
|
| 21 |
+
flash_attention: true
|
| 22 |
+
attention_dropout: 0.0
|
| 23 |
+
multi_query_attention: null
|
| 24 |
+
attention_layer_norm: true
|
| 25 |
+
residual_dropout: 0.0
|
| 26 |
+
embedding_dropout: 0.0
|
| 27 |
+
embedding_layer_norm: false
|
| 28 |
+
layer_norm_type: rms
|
| 29 |
+
layer_norm_with_affine: true
|
| 30 |
+
layer_norm_eps: 1.0e-05
|
| 31 |
+
attention_layer_norm_with_affine: true
|
| 32 |
+
max_sequence_length: 4096
|
| 33 |
+
include_bias: false
|
| 34 |
+
bias_for_layer_norm: false
|
| 35 |
+
scale_logits: false
|
| 36 |
+
vocab_size: 50280
|
| 37 |
+
embedding_size: 50304
|
| 38 |
+
weight_tying: false
|
| 39 |
+
eos_token_id: 0
|
| 40 |
+
pad_token_id: 1
|
| 41 |
+
init_device: meta
|
| 42 |
+
init_fn: normal
|
| 43 |
+
init_std: 0.02
|
| 44 |
+
init_cutoff_factor: 3.0
|
| 45 |
+
precision: amp_bf16
|
| 46 |
+
moe_num_experts: 64
|
| 47 |
+
moe_top_k: 8
|
| 48 |
+
moe_mlp_impl: sparse
|
| 49 |
+
moe_log_expert_assignment: false
|
| 50 |
+
moe_shared_expert: false
|
| 51 |
+
moe_lbl_in_fp32: false
|
| 52 |
+
moe_loss_weight: 0.01
|
| 53 |
+
moe_zloss_weight: 0.001
|
| 54 |
+
moe_dropless: false
|
| 55 |
+
moe_capacity_factor: 1.25
|
| 56 |
+
himoe_enabled: false
|
| 57 |
+
himoe_n_groups: 4
|
| 58 |
+
himoe_top_k_per_group: 2
|
| 59 |
+
himoe_lambda1: 0.01
|
| 60 |
+
himoe_lambda2: 0.1
|
| 61 |
+
scale_emb_init: false
|
| 62 |
+
emb_init_std: null
|
| 63 |
+
norm_after: false
|
| 64 |
+
optimizer:
|
| 65 |
+
name: adamw
|
| 66 |
+
learning_rate: 0.0004
|
| 67 |
+
weight_decay: 0.1
|
| 68 |
+
betas:
|
| 69 |
+
- 0.9
|
| 70 |
+
- 0.95
|
| 71 |
+
eps: 1.0e-08
|
| 72 |
+
no_decay_norm_and_bias: null
|
| 73 |
+
selective_updates: false
|
| 74 |
+
decay_norm_and_bias: true
|
| 75 |
+
decay_embeddings: true
|
| 76 |
+
metrics_log_interval: 10
|
| 77 |
+
record_update_metrics: false
|
| 78 |
+
scheduler:
|
| 79 |
+
name: cosine_with_warmup
|
| 80 |
+
units: tokens
|
| 81 |
+
t_warmup: 1405296000
|
| 82 |
+
t_max: 58540000000
|
| 83 |
+
alpha_f: 0.1
|
| 84 |
+
grad_clip_warmup_steps: null
|
| 85 |
+
grad_clip_warmup_factor: null
|
| 86 |
+
warmup_min_lr: null
|
| 87 |
+
data:
|
| 88 |
+
paths:
|
| 89 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy
|
| 90 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy
|
| 91 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy
|
| 92 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy
|
| 93 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy
|
| 94 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy
|
| 95 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy
|
| 96 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy
|
| 97 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy
|
| 98 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy
|
| 99 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy
|
| 100 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy
|
| 101 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy
|
| 102 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-13-00000.npy
|
| 103 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-14-00000.npy
|
| 104 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-15-00000.npy
|
| 105 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-000-00000.npy
|
| 106 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-001-00000.npy
|
| 107 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-002-00000.npy
|
| 108 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-003-00000.npy
|
| 109 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-004-00000.npy
|
| 110 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-005-00000.npy
|
| 111 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-006-00000.npy
|
| 112 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-007-00000.npy
|
| 113 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-008-00000.npy
|
| 114 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-009-00000.npy
|
| 115 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-010-00000.npy
|
| 116 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-011-00000.npy
|
| 117 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-012-00000.npy
|
| 118 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-013-00000.npy
|
| 119 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy
|
| 120 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy
|
| 121 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy
|
| 122 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy
|
| 123 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy
|
| 124 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy
|
| 125 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy
|
| 126 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy
|
| 127 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy
|
| 128 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy
|
| 129 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy
|
| 130 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy
|
| 131 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy
|
| 132 |
+
memmap_dtype: uint16
|
| 133 |
+
datasets: null
|
| 134 |
+
label_mask_paths: null
|
| 135 |
+
pad_direction: right
|
| 136 |
+
generate_attention_mask: false
|
| 137 |
+
generate_doc_lengths: false
|
| 138 |
+
num_workers: 8
|
| 139 |
+
drop_last: true
|
| 140 |
+
pin_memory: true
|
| 141 |
+
prefetch_factor: 8
|
| 142 |
+
persistent_workers: true
|
| 143 |
+
timeout: 0
|
| 144 |
+
seed: null
|
| 145 |
+
instance_filter:
|
| 146 |
+
repetition_max_period: 13
|
| 147 |
+
repetition_min_period: 1
|
| 148 |
+
repetition_max_count: 32
|
| 149 |
+
restore_dataloader: true
|
| 150 |
+
fast_forward_batches: null
|
| 151 |
+
evaluators:
|
| 152 |
+
- label: all-small-ppl-validation
|
| 153 |
+
type: lm
|
| 154 |
+
data:
|
| 155 |
+
paths: null
|
| 156 |
+
memmap_dtype: uint16
|
| 157 |
+
datasets:
|
| 158 |
+
c4_en-validation:
|
| 159 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/c4_en/val/part-0-00000.npy
|
| 160 |
+
dolma_books-validation:
|
| 161 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_books/val/part-0-00000.npy
|
| 162 |
+
dolma_common-crawl-validation:
|
| 163 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_common-crawl/val/part-0-00000.npy
|
| 164 |
+
dolma_pes2o-validation:
|
| 165 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_pes2o/val/part-0-00000.npy
|
| 166 |
+
dolma_reddit-validation:
|
| 167 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_reddit/val/part-0-00000.npy
|
| 168 |
+
dolma_stack-validation:
|
| 169 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_stack/val/part-0-00000.npy
|
| 170 |
+
dolma_wiki-validation:
|
| 171 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_wiki/val/part-0-00000.npy
|
| 172 |
+
ice-validation:
|
| 173 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/ice/val/part-0-00000.npy
|
| 174 |
+
m2d2_s2orc-validation:
|
| 175 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/m2d2_s2orc/val/part-0-00000.npy
|
| 176 |
+
pile-validation:
|
| 177 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/pile/val/part-0-00000.npy
|
| 178 |
+
wikitext_103-validation:
|
| 179 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/wikitext_103/val/part-0-00000.npy
|
| 180 |
+
label_mask_paths: null
|
| 181 |
+
pad_direction: right
|
| 182 |
+
generate_attention_mask: false
|
| 183 |
+
generate_doc_lengths: false
|
| 184 |
+
num_workers: 0
|
| 185 |
+
drop_last: true
|
| 186 |
+
pin_memory: false
|
| 187 |
+
prefetch_factor: null
|
| 188 |
+
persistent_workers: false
|
| 189 |
+
timeout: 0
|
| 190 |
+
seed: null
|
| 191 |
+
instance_filter: null
|
| 192 |
+
device_eval_batch_size: null
|
| 193 |
+
subset_num_batches: null
|
| 194 |
+
- label: piqa
|
| 195 |
+
type: downstream
|
| 196 |
+
data:
|
| 197 |
+
paths: null
|
| 198 |
+
memmap_dtype: uint16
|
| 199 |
+
datasets: null
|
| 200 |
+
label_mask_paths: null
|
| 201 |
+
pad_direction: right
|
| 202 |
+
generate_attention_mask: false
|
| 203 |
+
generate_doc_lengths: false
|
| 204 |
+
num_workers: 0
|
| 205 |
+
drop_last: false
|
| 206 |
+
pin_memory: false
|
| 207 |
+
prefetch_factor: null
|
| 208 |
+
persistent_workers: false
|
| 209 |
+
timeout: 0
|
| 210 |
+
seed: null
|
| 211 |
+
instance_filter: null
|
| 212 |
+
device_eval_batch_size: null
|
| 213 |
+
subset_num_batches: null
|
| 214 |
+
- label: hellaswag
|
| 215 |
+
type: downstream
|
| 216 |
+
data:
|
| 217 |
+
paths: null
|
| 218 |
+
memmap_dtype: uint16
|
| 219 |
+
datasets: null
|
| 220 |
+
label_mask_paths: null
|
| 221 |
+
pad_direction: right
|
| 222 |
+
generate_attention_mask: false
|
| 223 |
+
generate_doc_lengths: false
|
| 224 |
+
num_workers: 0
|
| 225 |
+
drop_last: false
|
| 226 |
+
pin_memory: false
|
| 227 |
+
prefetch_factor: null
|
| 228 |
+
persistent_workers: false
|
| 229 |
+
timeout: 0
|
| 230 |
+
seed: null
|
| 231 |
+
instance_filter: null
|
| 232 |
+
device_eval_batch_size: null
|
| 233 |
+
subset_num_batches: null
|
| 234 |
+
- label: winogrande
|
| 235 |
+
type: downstream
|
| 236 |
+
data:
|
| 237 |
+
paths: null
|
| 238 |
+
memmap_dtype: uint16
|
| 239 |
+
datasets: null
|
| 240 |
+
label_mask_paths: null
|
| 241 |
+
pad_direction: right
|
| 242 |
+
generate_attention_mask: false
|
| 243 |
+
generate_doc_lengths: false
|
| 244 |
+
num_workers: 0
|
| 245 |
+
drop_last: false
|
| 246 |
+
pin_memory: false
|
| 247 |
+
prefetch_factor: null
|
| 248 |
+
persistent_workers: false
|
| 249 |
+
timeout: 0
|
| 250 |
+
seed: null
|
| 251 |
+
instance_filter: null
|
| 252 |
+
device_eval_batch_size: null
|
| 253 |
+
subset_num_batches: null
|
| 254 |
+
- label: openbook_qa
|
| 255 |
+
type: downstream
|
| 256 |
+
data:
|
| 257 |
+
paths: null
|
| 258 |
+
memmap_dtype: uint16
|
| 259 |
+
datasets: null
|
| 260 |
+
label_mask_paths: null
|
| 261 |
+
pad_direction: right
|
| 262 |
+
generate_attention_mask: false
|
| 263 |
+
generate_doc_lengths: false
|
| 264 |
+
num_workers: 0
|
| 265 |
+
drop_last: false
|
| 266 |
+
pin_memory: false
|
| 267 |
+
prefetch_factor: null
|
| 268 |
+
persistent_workers: false
|
| 269 |
+
timeout: 0
|
| 270 |
+
seed: null
|
| 271 |
+
instance_filter: null
|
| 272 |
+
device_eval_batch_size: null
|
| 273 |
+
subset_num_batches: null
|
| 274 |
+
- label: boolq
|
| 275 |
+
type: downstream
|
| 276 |
+
data:
|
| 277 |
+
paths: null
|
| 278 |
+
memmap_dtype: uint16
|
| 279 |
+
datasets: null
|
| 280 |
+
label_mask_paths: null
|
| 281 |
+
pad_direction: right
|
| 282 |
+
generate_attention_mask: false
|
| 283 |
+
generate_doc_lengths: false
|
| 284 |
+
num_workers: 0
|
| 285 |
+
drop_last: false
|
| 286 |
+
pin_memory: false
|
| 287 |
+
prefetch_factor: null
|
| 288 |
+
persistent_workers: false
|
| 289 |
+
timeout: 0
|
| 290 |
+
seed: null
|
| 291 |
+
instance_filter: null
|
| 292 |
+
device_eval_batch_size: null
|
| 293 |
+
subset_num_batches: null
|
| 294 |
+
- label: sciq
|
| 295 |
+
type: downstream
|
| 296 |
+
data:
|
| 297 |
+
paths: null
|
| 298 |
+
memmap_dtype: uint16
|
| 299 |
+
datasets: null
|
| 300 |
+
label_mask_paths: null
|
| 301 |
+
pad_direction: right
|
| 302 |
+
generate_attention_mask: false
|
| 303 |
+
generate_doc_lengths: false
|
| 304 |
+
num_workers: 0
|
| 305 |
+
drop_last: false
|
| 306 |
+
pin_memory: false
|
| 307 |
+
prefetch_factor: null
|
| 308 |
+
persistent_workers: false
|
| 309 |
+
timeout: 0
|
| 310 |
+
seed: null
|
| 311 |
+
instance_filter: null
|
| 312 |
+
device_eval_batch_size: null
|
| 313 |
+
subset_num_batches: null
|
| 314 |
+
- label: arc_easy
|
| 315 |
+
type: downstream
|
| 316 |
+
data:
|
| 317 |
+
paths: null
|
| 318 |
+
memmap_dtype: uint16
|
| 319 |
+
datasets: null
|
| 320 |
+
label_mask_paths: null
|
| 321 |
+
pad_direction: right
|
| 322 |
+
generate_attention_mask: false
|
| 323 |
+
generate_doc_lengths: false
|
| 324 |
+
num_workers: 0
|
| 325 |
+
drop_last: false
|
| 326 |
+
pin_memory: false
|
| 327 |
+
prefetch_factor: null
|
| 328 |
+
persistent_workers: false
|
| 329 |
+
timeout: 0
|
| 330 |
+
seed: null
|
| 331 |
+
instance_filter: null
|
| 332 |
+
device_eval_batch_size: null
|
| 333 |
+
subset_num_batches: null
|
| 334 |
+
- label: arc_challenge
|
| 335 |
+
type: downstream
|
| 336 |
+
data:
|
| 337 |
+
paths: null
|
| 338 |
+
memmap_dtype: uint16
|
| 339 |
+
datasets: null
|
| 340 |
+
label_mask_paths: null
|
| 341 |
+
pad_direction: right
|
| 342 |
+
generate_attention_mask: false
|
| 343 |
+
generate_doc_lengths: false
|
| 344 |
+
num_workers: 0
|
| 345 |
+
drop_last: false
|
| 346 |
+
pin_memory: false
|
| 347 |
+
prefetch_factor: null
|
| 348 |
+
persistent_workers: false
|
| 349 |
+
timeout: 0
|
| 350 |
+
seed: null
|
| 351 |
+
instance_filter: null
|
| 352 |
+
device_eval_batch_size: null
|
| 353 |
+
subset_num_batches: null
|
| 354 |
+
- label: copa
|
| 355 |
+
type: downstream
|
| 356 |
+
data:
|
| 357 |
+
paths: null
|
| 358 |
+
memmap_dtype: uint16
|
| 359 |
+
datasets: null
|
| 360 |
+
label_mask_paths: null
|
| 361 |
+
pad_direction: right
|
| 362 |
+
generate_attention_mask: false
|
| 363 |
+
generate_doc_lengths: false
|
| 364 |
+
num_workers: 0
|
| 365 |
+
drop_last: false
|
| 366 |
+
pin_memory: false
|
| 367 |
+
prefetch_factor: null
|
| 368 |
+
persistent_workers: false
|
| 369 |
+
timeout: 0
|
| 370 |
+
seed: null
|
| 371 |
+
instance_filter: null
|
| 372 |
+
device_eval_batch_size: null
|
| 373 |
+
subset_num_batches: null
|
| 374 |
+
- label: commonsense_qa
|
| 375 |
+
type: downstream
|
| 376 |
+
data:
|
| 377 |
+
paths: null
|
| 378 |
+
memmap_dtype: uint16
|
| 379 |
+
datasets: null
|
| 380 |
+
label_mask_paths: null
|
| 381 |
+
pad_direction: right
|
| 382 |
+
generate_attention_mask: false
|
| 383 |
+
generate_doc_lengths: false
|
| 384 |
+
num_workers: 0
|
| 385 |
+
drop_last: false
|
| 386 |
+
pin_memory: false
|
| 387 |
+
prefetch_factor: null
|
| 388 |
+
persistent_workers: false
|
| 389 |
+
timeout: 0
|
| 390 |
+
seed: null
|
| 391 |
+
instance_filter: null
|
| 392 |
+
device_eval_batch_size: null
|
| 393 |
+
subset_num_batches: null
|
| 394 |
+
- label: social_iqa
|
| 395 |
+
type: downstream
|
| 396 |
+
data:
|
| 397 |
+
paths: null
|
| 398 |
+
memmap_dtype: uint16
|
| 399 |
+
datasets: null
|
| 400 |
+
label_mask_paths: null
|
| 401 |
+
pad_direction: right
|
| 402 |
+
generate_attention_mask: false
|
| 403 |
+
generate_doc_lengths: false
|
| 404 |
+
num_workers: 0
|
| 405 |
+
drop_last: false
|
| 406 |
+
pin_memory: false
|
| 407 |
+
prefetch_factor: null
|
| 408 |
+
persistent_workers: false
|
| 409 |
+
timeout: 0
|
| 410 |
+
seed: null
|
| 411 |
+
instance_filter: null
|
| 412 |
+
device_eval_batch_size: null
|
| 413 |
+
subset_num_batches: null
|
| 414 |
+
- label: mmlu_stem_var
|
| 415 |
+
type: downstream
|
| 416 |
+
data:
|
| 417 |
+
paths: null
|
| 418 |
+
memmap_dtype: uint16
|
| 419 |
+
datasets: null
|
| 420 |
+
label_mask_paths: null
|
| 421 |
+
pad_direction: right
|
| 422 |
+
generate_attention_mask: false
|
| 423 |
+
generate_doc_lengths: false
|
| 424 |
+
num_workers: 0
|
| 425 |
+
drop_last: false
|
| 426 |
+
pin_memory: false
|
| 427 |
+
prefetch_factor: null
|
| 428 |
+
persistent_workers: false
|
| 429 |
+
timeout: 0
|
| 430 |
+
seed: null
|
| 431 |
+
instance_filter: null
|
| 432 |
+
device_eval_batch_size: null
|
| 433 |
+
subset_num_batches: null
|
| 434 |
+
- label: mmlu_humanities_var
|
| 435 |
+
type: downstream
|
| 436 |
+
data:
|
| 437 |
+
paths: null
|
| 438 |
+
memmap_dtype: uint16
|
| 439 |
+
datasets: null
|
| 440 |
+
label_mask_paths: null
|
| 441 |
+
pad_direction: right
|
| 442 |
+
generate_attention_mask: false
|
| 443 |
+
generate_doc_lengths: false
|
| 444 |
+
num_workers: 0
|
| 445 |
+
drop_last: false
|
| 446 |
+
pin_memory: false
|
| 447 |
+
prefetch_factor: null
|
| 448 |
+
persistent_workers: false
|
| 449 |
+
timeout: 0
|
| 450 |
+
seed: null
|
| 451 |
+
instance_filter: null
|
| 452 |
+
device_eval_batch_size: null
|
| 453 |
+
subset_num_batches: null
|
| 454 |
+
- label: mmlu_social_sciences_var
|
| 455 |
+
type: downstream
|
| 456 |
+
data:
|
| 457 |
+
paths: null
|
| 458 |
+
memmap_dtype: uint16
|
| 459 |
+
datasets: null
|
| 460 |
+
label_mask_paths: null
|
| 461 |
+
pad_direction: right
|
| 462 |
+
generate_attention_mask: false
|
| 463 |
+
generate_doc_lengths: false
|
| 464 |
+
num_workers: 0
|
| 465 |
+
drop_last: false
|
| 466 |
+
pin_memory: false
|
| 467 |
+
prefetch_factor: null
|
| 468 |
+
persistent_workers: false
|
| 469 |
+
timeout: 0
|
| 470 |
+
seed: null
|
| 471 |
+
instance_filter: null
|
| 472 |
+
device_eval_batch_size: null
|
| 473 |
+
subset_num_batches: null
|
| 474 |
+
- label: mmlu_other_var
|
| 475 |
+
type: downstream
|
| 476 |
+
data:
|
| 477 |
+
paths: null
|
| 478 |
+
memmap_dtype: uint16
|
| 479 |
+
datasets: null
|
| 480 |
+
label_mask_paths: null
|
| 481 |
+
pad_direction: right
|
| 482 |
+
generate_attention_mask: false
|
| 483 |
+
generate_doc_lengths: false
|
| 484 |
+
num_workers: 0
|
| 485 |
+
drop_last: false
|
| 486 |
+
pin_memory: false
|
| 487 |
+
prefetch_factor: null
|
| 488 |
+
persistent_workers: false
|
| 489 |
+
timeout: 0
|
| 490 |
+
seed: null
|
| 491 |
+
instance_filter: null
|
| 492 |
+
device_eval_batch_size: null
|
| 493 |
+
subset_num_batches: null
|
| 494 |
+
- label: mmlu_stem_mc_5shot
|
| 495 |
+
type: downstream
|
| 496 |
+
data:
|
| 497 |
+
paths: null
|
| 498 |
+
memmap_dtype: uint16
|
| 499 |
+
datasets: null
|
| 500 |
+
label_mask_paths: null
|
| 501 |
+
pad_direction: right
|
| 502 |
+
generate_attention_mask: false
|
| 503 |
+
generate_doc_lengths: false
|
| 504 |
+
num_workers: 0
|
| 505 |
+
drop_last: false
|
| 506 |
+
pin_memory: false
|
| 507 |
+
prefetch_factor: null
|
| 508 |
+
persistent_workers: false
|
| 509 |
+
timeout: 0
|
| 510 |
+
seed: null
|
| 511 |
+
instance_filter: null
|
| 512 |
+
device_eval_batch_size: null
|
| 513 |
+
subset_num_batches: null
|
| 514 |
+
- label: mmlu_humanities_mc_5shot
|
| 515 |
+
type: downstream
|
| 516 |
+
data:
|
| 517 |
+
paths: null
|
| 518 |
+
memmap_dtype: uint16
|
| 519 |
+
datasets: null
|
| 520 |
+
label_mask_paths: null
|
| 521 |
+
pad_direction: right
|
| 522 |
+
generate_attention_mask: false
|
| 523 |
+
generate_doc_lengths: false
|
| 524 |
+
num_workers: 0
|
| 525 |
+
drop_last: false
|
| 526 |
+
pin_memory: false
|
| 527 |
+
prefetch_factor: null
|
| 528 |
+
persistent_workers: false
|
| 529 |
+
timeout: 0
|
| 530 |
+
seed: null
|
| 531 |
+
instance_filter: null
|
| 532 |
+
device_eval_batch_size: null
|
| 533 |
+
subset_num_batches: null
|
| 534 |
+
- label: mmlu_social_sciences_mc_5shot
|
| 535 |
+
type: downstream
|
| 536 |
+
data:
|
| 537 |
+
paths: null
|
| 538 |
+
memmap_dtype: uint16
|
| 539 |
+
datasets: null
|
| 540 |
+
label_mask_paths: null
|
| 541 |
+
pad_direction: right
|
| 542 |
+
generate_attention_mask: false
|
| 543 |
+
generate_doc_lengths: false
|
| 544 |
+
num_workers: 0
|
| 545 |
+
drop_last: false
|
| 546 |
+
pin_memory: false
|
| 547 |
+
prefetch_factor: null
|
| 548 |
+
persistent_workers: false
|
| 549 |
+
timeout: 0
|
| 550 |
+
seed: null
|
| 551 |
+
instance_filter: null
|
| 552 |
+
device_eval_batch_size: null
|
| 553 |
+
subset_num_batches: null
|
| 554 |
+
- label: mmlu_other_mc_5shot
|
| 555 |
+
type: downstream
|
| 556 |
+
data:
|
| 557 |
+
paths: null
|
| 558 |
+
memmap_dtype: uint16
|
| 559 |
+
datasets: null
|
| 560 |
+
label_mask_paths: null
|
| 561 |
+
pad_direction: right
|
| 562 |
+
generate_attention_mask: false
|
| 563 |
+
generate_doc_lengths: false
|
| 564 |
+
num_workers: 0
|
| 565 |
+
drop_last: false
|
| 566 |
+
pin_memory: false
|
| 567 |
+
prefetch_factor: null
|
| 568 |
+
persistent_workers: false
|
| 569 |
+
timeout: 0
|
| 570 |
+
seed: null
|
| 571 |
+
instance_filter: null
|
| 572 |
+
device_eval_batch_size: null
|
| 573 |
+
subset_num_batches: null
|
| 574 |
+
- label: mmlu_stem_mc_5shot_test
|
| 575 |
+
type: downstream
|
| 576 |
+
data:
|
| 577 |
+
paths: null
|
| 578 |
+
memmap_dtype: uint16
|
| 579 |
+
datasets: null
|
| 580 |
+
label_mask_paths: null
|
| 581 |
+
pad_direction: right
|
| 582 |
+
generate_attention_mask: false
|
| 583 |
+
generate_doc_lengths: false
|
| 584 |
+
num_workers: 0
|
| 585 |
+
drop_last: false
|
| 586 |
+
pin_memory: false
|
| 587 |
+
prefetch_factor: null
|
| 588 |
+
persistent_workers: false
|
| 589 |
+
timeout: 0
|
| 590 |
+
seed: null
|
| 591 |
+
instance_filter: null
|
| 592 |
+
device_eval_batch_size: null
|
| 593 |
+
subset_num_batches: null
|
| 594 |
+
- label: mmlu_humanities_mc_5shot_test
|
| 595 |
+
type: downstream
|
| 596 |
+
data:
|
| 597 |
+
paths: null
|
| 598 |
+
memmap_dtype: uint16
|
| 599 |
+
datasets: null
|
| 600 |
+
label_mask_paths: null
|
| 601 |
+
pad_direction: right
|
| 602 |
+
generate_attention_mask: false
|
| 603 |
+
generate_doc_lengths: false
|
| 604 |
+
num_workers: 0
|
| 605 |
+
drop_last: false
|
| 606 |
+
pin_memory: false
|
| 607 |
+
prefetch_factor: null
|
| 608 |
+
persistent_workers: false
|
| 609 |
+
timeout: 0
|
| 610 |
+
seed: null
|
| 611 |
+
instance_filter: null
|
| 612 |
+
device_eval_batch_size: null
|
| 613 |
+
subset_num_batches: null
|
| 614 |
+
- label: mmlu_social_sciences_mc_5shot_test
|
| 615 |
+
type: downstream
|
| 616 |
+
data:
|
| 617 |
+
paths: null
|
| 618 |
+
memmap_dtype: uint16
|
| 619 |
+
datasets: null
|
| 620 |
+
label_mask_paths: null
|
| 621 |
+
pad_direction: right
|
| 622 |
+
generate_attention_mask: false
|
| 623 |
+
generate_doc_lengths: false
|
| 624 |
+
num_workers: 0
|
| 625 |
+
drop_last: false
|
| 626 |
+
pin_memory: false
|
| 627 |
+
prefetch_factor: null
|
| 628 |
+
persistent_workers: false
|
| 629 |
+
timeout: 0
|
| 630 |
+
seed: null
|
| 631 |
+
instance_filter: null
|
| 632 |
+
device_eval_batch_size: null
|
| 633 |
+
subset_num_batches: null
|
| 634 |
+
- label: mmlu_other_mc_5shot_test
|
| 635 |
+
type: downstream
|
| 636 |
+
data:
|
| 637 |
+
paths: null
|
| 638 |
+
memmap_dtype: uint16
|
| 639 |
+
datasets: null
|
| 640 |
+
label_mask_paths: null
|
| 641 |
+
pad_direction: right
|
| 642 |
+
generate_attention_mask: false
|
| 643 |
+
generate_doc_lengths: false
|
| 644 |
+
num_workers: 0
|
| 645 |
+
drop_last: false
|
| 646 |
+
pin_memory: false
|
| 647 |
+
prefetch_factor: null
|
| 648 |
+
persistent_workers: false
|
| 649 |
+
timeout: 0
|
| 650 |
+
seed: null
|
| 651 |
+
instance_filter: null
|
| 652 |
+
device_eval_batch_size: null
|
| 653 |
+
subset_num_batches: null
|
| 654 |
+
eval_interval: 2000
|
| 655 |
+
tokenizer:
|
| 656 |
+
identifier: tokenizers/allenai_gpt-neox-olmo-dolma-v1_5.json
|
| 657 |
+
truncate_direction: right
|
| 658 |
+
save_folder: runs/olmoe-7b-29b-2ep
|
| 659 |
+
remote_save_folder: null
|
| 660 |
+
canceled_check_interval: 50
|
| 661 |
+
save_interval: 2000
|
| 662 |
+
save_interval_unsharded: null
|
| 663 |
+
save_interval_ephemeral: null
|
| 664 |
+
save_num_checkpoints_to_keep: 5
|
| 665 |
+
save_num_unsharded_checkpoints_to_keep: -1
|
| 666 |
+
save_overwrite: true
|
| 667 |
+
force_save_unsharded: false
|
| 668 |
+
no_pre_train_checkpoint: true
|
| 669 |
+
load_path: null
|
| 670 |
+
load_path_sharded_checkpointer: null
|
| 671 |
+
try_load_latest_save: false
|
| 672 |
+
reset_optimizer_state: false
|
| 673 |
+
reset_trainer_state: false
|
| 674 |
+
sharded_checkpointer: olmo_core
|
| 675 |
+
new_style_checkpoints: null
|
| 676 |
+
max_duration: 2ep
|
| 677 |
+
global_train_batch_size: 1024
|
| 678 |
+
device_train_batch_size: 256
|
| 679 |
+
device_train_microbatch_size: 2
|
| 680 |
+
device_eval_batch_size: 1
|
| 681 |
+
eval_subset_num_batches: -1
|
| 682 |
+
eval_on_load: false
|
| 683 |
+
device_train_grad_accum: 128
|
| 684 |
+
max_grad_norm: 1.0
|
| 685 |
+
max_grad_norm_ratio: null
|
| 686 |
+
precision: amp_bf16
|
| 687 |
+
wandb:
|
| 688 |
+
project: hiomoe
|
| 689 |
+
entity: bulgakov
|
| 690 |
+
group: 500gb-test
|
| 691 |
+
name: olmoe-7b-29b-2ep
|
| 692 |
+
tags:
|
| 693 |
+
- watching
|
| 694 |
+
log_artifacts: false
|
| 695 |
+
rank_zero_only: true
|
| 696 |
+
log_interval: 1
|
| 697 |
+
speed_monitor:
|
| 698 |
+
window_size: 1
|
| 699 |
+
gpu_flops_available: null
|
| 700 |
+
console_log_interval: 1
|
| 701 |
+
gen1_gc_interval: 1
|
| 702 |
+
compile: null
|
| 703 |
+
distributed_strategy: fsdp
|
| 704 |
+
fsdp:
|
| 705 |
+
use_orig_params: true
|
| 706 |
+
sharding_strategy: FULL_SHARD
|
| 707 |
+
wrapping_strategy: by_block
|
| 708 |
+
precision: mixed
|
| 709 |
+
hybrid_sharding_num_model_replicas: null
|
| 710 |
+
ddp: null
|
| 711 |
+
softmax_auxiliary_loss: false
|
| 712 |
+
auxiliary_loss_multiplier: 0.0001
|
| 713 |
+
time_limit: null
|
| 714 |
+
extra_steps_after_cancel: 10
|
| 715 |
+
early_stopping_factor: null
|
| 716 |
+
save_data_indices: true
|
| 717 |
+
python_profiling: false
|
| 718 |
+
torch_profiling: false
|
| 719 |
+
stop_at: 13964
|
| 720 |
+
stop_after: null
|
| 721 |
+
activation_checkpointing: fine_grained
|
| 722 |
+
fused_loss: null
|
| 723 |
+
hf_datasets_cache_dir: null
|
| 724 |
+
module_outputs_save_steps: null
|
step6000/model/metadata.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"tensors": {"transformer.wte.weight": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 51511808]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[51511808, 103022592]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.ln_f.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 1024]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[1024, 2048]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.ff_out.weight": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 51510784]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[51510784, 103022592]], "local_shape": null, "global_offset": null}}}}}
|
step6000/optim/metadata.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
step6000/train/rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c413c99a6338e5cb932add36c4070b4eecbe52fa500afa3729cc694c10f2dfbe
|
| 3 |
+
size 14933
|
step6000/train/rank1.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ee5f375854ccf6d5598d3fb3cb6928ca66af8b3da0f3f187a586256470d63c06
|
| 3 |
+
size 14933
|
step6000/train/rank2.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ba353b8a18abd862774ef6f1b0ba39e34dd4b6609305b8d8618374922418ef32
|
| 3 |
+
size 14933
|
step6000/train/rank3.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:98002e33db5c87ed9e1d7a20d657971aa4046435183ff43ae38cd434e93a92d1
|
| 3 |
+
size 14933
|
step8000/config.yaml
ADDED
|
@@ -0,0 +1,724 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
run_name: olmoe-7b-29b-2ep
|
| 2 |
+
seed: 6198
|
| 3 |
+
epoch: null
|
| 4 |
+
dry_run: false
|
| 5 |
+
model:
|
| 6 |
+
d_model: 2048
|
| 7 |
+
n_heads: 16
|
| 8 |
+
n_kv_heads: null
|
| 9 |
+
clip_qkv: null
|
| 10 |
+
n_layers: 16
|
| 11 |
+
mlp_ratio: 1
|
| 12 |
+
mlp_hidden_size: null
|
| 13 |
+
activation_type: swiglu
|
| 14 |
+
block_type: moe
|
| 15 |
+
block_group_size: 1
|
| 16 |
+
alibi: false
|
| 17 |
+
alibi_bias_max: 8.0
|
| 18 |
+
rope: true
|
| 19 |
+
rope_full_precision: true
|
| 20 |
+
rope_theta: 10000
|
| 21 |
+
flash_attention: true
|
| 22 |
+
attention_dropout: 0.0
|
| 23 |
+
multi_query_attention: null
|
| 24 |
+
attention_layer_norm: true
|
| 25 |
+
residual_dropout: 0.0
|
| 26 |
+
embedding_dropout: 0.0
|
| 27 |
+
embedding_layer_norm: false
|
| 28 |
+
layer_norm_type: rms
|
| 29 |
+
layer_norm_with_affine: true
|
| 30 |
+
layer_norm_eps: 1.0e-05
|
| 31 |
+
attention_layer_norm_with_affine: true
|
| 32 |
+
max_sequence_length: 4096
|
| 33 |
+
include_bias: false
|
| 34 |
+
bias_for_layer_norm: false
|
| 35 |
+
scale_logits: false
|
| 36 |
+
vocab_size: 50280
|
| 37 |
+
embedding_size: 50304
|
| 38 |
+
weight_tying: false
|
| 39 |
+
eos_token_id: 0
|
| 40 |
+
pad_token_id: 1
|
| 41 |
+
init_device: meta
|
| 42 |
+
init_fn: normal
|
| 43 |
+
init_std: 0.02
|
| 44 |
+
init_cutoff_factor: 3.0
|
| 45 |
+
precision: amp_bf16
|
| 46 |
+
moe_num_experts: 64
|
| 47 |
+
moe_top_k: 8
|
| 48 |
+
moe_mlp_impl: sparse
|
| 49 |
+
moe_log_expert_assignment: false
|
| 50 |
+
moe_shared_expert: false
|
| 51 |
+
moe_lbl_in_fp32: false
|
| 52 |
+
moe_loss_weight: 0.01
|
| 53 |
+
moe_zloss_weight: 0.001
|
| 54 |
+
moe_dropless: false
|
| 55 |
+
moe_capacity_factor: 1.25
|
| 56 |
+
himoe_enabled: false
|
| 57 |
+
himoe_n_groups: 4
|
| 58 |
+
himoe_top_k_per_group: 2
|
| 59 |
+
himoe_lambda1: 0.01
|
| 60 |
+
himoe_lambda2: 0.1
|
| 61 |
+
scale_emb_init: false
|
| 62 |
+
emb_init_std: null
|
| 63 |
+
norm_after: false
|
| 64 |
+
optimizer:
|
| 65 |
+
name: adamw
|
| 66 |
+
learning_rate: 0.0004
|
| 67 |
+
weight_decay: 0.1
|
| 68 |
+
betas:
|
| 69 |
+
- 0.9
|
| 70 |
+
- 0.95
|
| 71 |
+
eps: 1.0e-08
|
| 72 |
+
no_decay_norm_and_bias: null
|
| 73 |
+
selective_updates: false
|
| 74 |
+
decay_norm_and_bias: true
|
| 75 |
+
decay_embeddings: true
|
| 76 |
+
metrics_log_interval: 10
|
| 77 |
+
record_update_metrics: false
|
| 78 |
+
scheduler:
|
| 79 |
+
name: cosine_with_warmup
|
| 80 |
+
units: tokens
|
| 81 |
+
t_warmup: 1405296000
|
| 82 |
+
t_max: 58540000000
|
| 83 |
+
alpha_f: 0.1
|
| 84 |
+
grad_clip_warmup_steps: null
|
| 85 |
+
grad_clip_warmup_factor: null
|
| 86 |
+
warmup_min_lr: null
|
| 87 |
+
data:
|
| 88 |
+
paths:
|
| 89 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy
|
| 90 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy
|
| 91 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy
|
| 92 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy
|
| 93 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy
|
| 94 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy
|
| 95 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy
|
| 96 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy
|
| 97 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy
|
| 98 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy
|
| 99 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy
|
| 100 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy
|
| 101 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy
|
| 102 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-13-00000.npy
|
| 103 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-14-00000.npy
|
| 104 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-15-00000.npy
|
| 105 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-000-00000.npy
|
| 106 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-001-00000.npy
|
| 107 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-002-00000.npy
|
| 108 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-003-00000.npy
|
| 109 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-004-00000.npy
|
| 110 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-005-00000.npy
|
| 111 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-006-00000.npy
|
| 112 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-007-00000.npy
|
| 113 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-008-00000.npy
|
| 114 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-009-00000.npy
|
| 115 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-010-00000.npy
|
| 116 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-011-00000.npy
|
| 117 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-012-00000.npy
|
| 118 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-013-00000.npy
|
| 119 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy
|
| 120 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy
|
| 121 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy
|
| 122 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy
|
| 123 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy
|
| 124 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy
|
| 125 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy
|
| 126 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy
|
| 127 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy
|
| 128 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy
|
| 129 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy
|
| 130 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy
|
| 131 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy
|
| 132 |
+
memmap_dtype: uint16
|
| 133 |
+
datasets: null
|
| 134 |
+
label_mask_paths: null
|
| 135 |
+
pad_direction: right
|
| 136 |
+
generate_attention_mask: false
|
| 137 |
+
generate_doc_lengths: false
|
| 138 |
+
num_workers: 8
|
| 139 |
+
drop_last: true
|
| 140 |
+
pin_memory: true
|
| 141 |
+
prefetch_factor: 8
|
| 142 |
+
persistent_workers: true
|
| 143 |
+
timeout: 0
|
| 144 |
+
seed: null
|
| 145 |
+
instance_filter:
|
| 146 |
+
repetition_max_period: 13
|
| 147 |
+
repetition_min_period: 1
|
| 148 |
+
repetition_max_count: 32
|
| 149 |
+
restore_dataloader: true
|
| 150 |
+
fast_forward_batches: null
|
| 151 |
+
evaluators:
|
| 152 |
+
- label: all-small-ppl-validation
|
| 153 |
+
type: lm
|
| 154 |
+
data:
|
| 155 |
+
paths: null
|
| 156 |
+
memmap_dtype: uint16
|
| 157 |
+
datasets:
|
| 158 |
+
c4_en-validation:
|
| 159 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/c4_en/val/part-0-00000.npy
|
| 160 |
+
dolma_books-validation:
|
| 161 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_books/val/part-0-00000.npy
|
| 162 |
+
dolma_common-crawl-validation:
|
| 163 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_common-crawl/val/part-0-00000.npy
|
| 164 |
+
dolma_pes2o-validation:
|
| 165 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_pes2o/val/part-0-00000.npy
|
| 166 |
+
dolma_reddit-validation:
|
| 167 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_reddit/val/part-0-00000.npy
|
| 168 |
+
dolma_stack-validation:
|
| 169 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_stack/val/part-0-00000.npy
|
| 170 |
+
dolma_wiki-validation:
|
| 171 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_wiki/val/part-0-00000.npy
|
| 172 |
+
ice-validation:
|
| 173 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/ice/val/part-0-00000.npy
|
| 174 |
+
m2d2_s2orc-validation:
|
| 175 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/m2d2_s2orc/val/part-0-00000.npy
|
| 176 |
+
pile-validation:
|
| 177 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/pile/val/part-0-00000.npy
|
| 178 |
+
wikitext_103-validation:
|
| 179 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/wikitext_103/val/part-0-00000.npy
|
| 180 |
+
label_mask_paths: null
|
| 181 |
+
pad_direction: right
|
| 182 |
+
generate_attention_mask: false
|
| 183 |
+
generate_doc_lengths: false
|
| 184 |
+
num_workers: 0
|
| 185 |
+
drop_last: true
|
| 186 |
+
pin_memory: false
|
| 187 |
+
prefetch_factor: null
|
| 188 |
+
persistent_workers: false
|
| 189 |
+
timeout: 0
|
| 190 |
+
seed: null
|
| 191 |
+
instance_filter: null
|
| 192 |
+
device_eval_batch_size: null
|
| 193 |
+
subset_num_batches: null
|
| 194 |
+
- label: piqa
|
| 195 |
+
type: downstream
|
| 196 |
+
data:
|
| 197 |
+
paths: null
|
| 198 |
+
memmap_dtype: uint16
|
| 199 |
+
datasets: null
|
| 200 |
+
label_mask_paths: null
|
| 201 |
+
pad_direction: right
|
| 202 |
+
generate_attention_mask: false
|
| 203 |
+
generate_doc_lengths: false
|
| 204 |
+
num_workers: 0
|
| 205 |
+
drop_last: false
|
| 206 |
+
pin_memory: false
|
| 207 |
+
prefetch_factor: null
|
| 208 |
+
persistent_workers: false
|
| 209 |
+
timeout: 0
|
| 210 |
+
seed: null
|
| 211 |
+
instance_filter: null
|
| 212 |
+
device_eval_batch_size: null
|
| 213 |
+
subset_num_batches: null
|
| 214 |
+
- label: hellaswag
|
| 215 |
+
type: downstream
|
| 216 |
+
data:
|
| 217 |
+
paths: null
|
| 218 |
+
memmap_dtype: uint16
|
| 219 |
+
datasets: null
|
| 220 |
+
label_mask_paths: null
|
| 221 |
+
pad_direction: right
|
| 222 |
+
generate_attention_mask: false
|
| 223 |
+
generate_doc_lengths: false
|
| 224 |
+
num_workers: 0
|
| 225 |
+
drop_last: false
|
| 226 |
+
pin_memory: false
|
| 227 |
+
prefetch_factor: null
|
| 228 |
+
persistent_workers: false
|
| 229 |
+
timeout: 0
|
| 230 |
+
seed: null
|
| 231 |
+
instance_filter: null
|
| 232 |
+
device_eval_batch_size: null
|
| 233 |
+
subset_num_batches: null
|
| 234 |
+
- label: winogrande
|
| 235 |
+
type: downstream
|
| 236 |
+
data:
|
| 237 |
+
paths: null
|
| 238 |
+
memmap_dtype: uint16
|
| 239 |
+
datasets: null
|
| 240 |
+
label_mask_paths: null
|
| 241 |
+
pad_direction: right
|
| 242 |
+
generate_attention_mask: false
|
| 243 |
+
generate_doc_lengths: false
|
| 244 |
+
num_workers: 0
|
| 245 |
+
drop_last: false
|
| 246 |
+
pin_memory: false
|
| 247 |
+
prefetch_factor: null
|
| 248 |
+
persistent_workers: false
|
| 249 |
+
timeout: 0
|
| 250 |
+
seed: null
|
| 251 |
+
instance_filter: null
|
| 252 |
+
device_eval_batch_size: null
|
| 253 |
+
subset_num_batches: null
|
| 254 |
+
- label: openbook_qa
|
| 255 |
+
type: downstream
|
| 256 |
+
data:
|
| 257 |
+
paths: null
|
| 258 |
+
memmap_dtype: uint16
|
| 259 |
+
datasets: null
|
| 260 |
+
label_mask_paths: null
|
| 261 |
+
pad_direction: right
|
| 262 |
+
generate_attention_mask: false
|
| 263 |
+
generate_doc_lengths: false
|
| 264 |
+
num_workers: 0
|
| 265 |
+
drop_last: false
|
| 266 |
+
pin_memory: false
|
| 267 |
+
prefetch_factor: null
|
| 268 |
+
persistent_workers: false
|
| 269 |
+
timeout: 0
|
| 270 |
+
seed: null
|
| 271 |
+
instance_filter: null
|
| 272 |
+
device_eval_batch_size: null
|
| 273 |
+
subset_num_batches: null
|
| 274 |
+
- label: boolq
|
| 275 |
+
type: downstream
|
| 276 |
+
data:
|
| 277 |
+
paths: null
|
| 278 |
+
memmap_dtype: uint16
|
| 279 |
+
datasets: null
|
| 280 |
+
label_mask_paths: null
|
| 281 |
+
pad_direction: right
|
| 282 |
+
generate_attention_mask: false
|
| 283 |
+
generate_doc_lengths: false
|
| 284 |
+
num_workers: 0
|
| 285 |
+
drop_last: false
|
| 286 |
+
pin_memory: false
|
| 287 |
+
prefetch_factor: null
|
| 288 |
+
persistent_workers: false
|
| 289 |
+
timeout: 0
|
| 290 |
+
seed: null
|
| 291 |
+
instance_filter: null
|
| 292 |
+
device_eval_batch_size: null
|
| 293 |
+
subset_num_batches: null
|
| 294 |
+
- label: sciq
|
| 295 |
+
type: downstream
|
| 296 |
+
data:
|
| 297 |
+
paths: null
|
| 298 |
+
memmap_dtype: uint16
|
| 299 |
+
datasets: null
|
| 300 |
+
label_mask_paths: null
|
| 301 |
+
pad_direction: right
|
| 302 |
+
generate_attention_mask: false
|
| 303 |
+
generate_doc_lengths: false
|
| 304 |
+
num_workers: 0
|
| 305 |
+
drop_last: false
|
| 306 |
+
pin_memory: false
|
| 307 |
+
prefetch_factor: null
|
| 308 |
+
persistent_workers: false
|
| 309 |
+
timeout: 0
|
| 310 |
+
seed: null
|
| 311 |
+
instance_filter: null
|
| 312 |
+
device_eval_batch_size: null
|
| 313 |
+
subset_num_batches: null
|
| 314 |
+
- label: arc_easy
|
| 315 |
+
type: downstream
|
| 316 |
+
data:
|
| 317 |
+
paths: null
|
| 318 |
+
memmap_dtype: uint16
|
| 319 |
+
datasets: null
|
| 320 |
+
label_mask_paths: null
|
| 321 |
+
pad_direction: right
|
| 322 |
+
generate_attention_mask: false
|
| 323 |
+
generate_doc_lengths: false
|
| 324 |
+
num_workers: 0
|
| 325 |
+
drop_last: false
|
| 326 |
+
pin_memory: false
|
| 327 |
+
prefetch_factor: null
|
| 328 |
+
persistent_workers: false
|
| 329 |
+
timeout: 0
|
| 330 |
+
seed: null
|
| 331 |
+
instance_filter: null
|
| 332 |
+
device_eval_batch_size: null
|
| 333 |
+
subset_num_batches: null
|
| 334 |
+
- label: arc_challenge
|
| 335 |
+
type: downstream
|
| 336 |
+
data:
|
| 337 |
+
paths: null
|
| 338 |
+
memmap_dtype: uint16
|
| 339 |
+
datasets: null
|
| 340 |
+
label_mask_paths: null
|
| 341 |
+
pad_direction: right
|
| 342 |
+
generate_attention_mask: false
|
| 343 |
+
generate_doc_lengths: false
|
| 344 |
+
num_workers: 0
|
| 345 |
+
drop_last: false
|
| 346 |
+
pin_memory: false
|
| 347 |
+
prefetch_factor: null
|
| 348 |
+
persistent_workers: false
|
| 349 |
+
timeout: 0
|
| 350 |
+
seed: null
|
| 351 |
+
instance_filter: null
|
| 352 |
+
device_eval_batch_size: null
|
| 353 |
+
subset_num_batches: null
|
| 354 |
+
- label: copa
|
| 355 |
+
type: downstream
|
| 356 |
+
data:
|
| 357 |
+
paths: null
|
| 358 |
+
memmap_dtype: uint16
|
| 359 |
+
datasets: null
|
| 360 |
+
label_mask_paths: null
|
| 361 |
+
pad_direction: right
|
| 362 |
+
generate_attention_mask: false
|
| 363 |
+
generate_doc_lengths: false
|
| 364 |
+
num_workers: 0
|
| 365 |
+
drop_last: false
|
| 366 |
+
pin_memory: false
|
| 367 |
+
prefetch_factor: null
|
| 368 |
+
persistent_workers: false
|
| 369 |
+
timeout: 0
|
| 370 |
+
seed: null
|
| 371 |
+
instance_filter: null
|
| 372 |
+
device_eval_batch_size: null
|
| 373 |
+
subset_num_batches: null
|
| 374 |
+
- label: commonsense_qa
|
| 375 |
+
type: downstream
|
| 376 |
+
data:
|
| 377 |
+
paths: null
|
| 378 |
+
memmap_dtype: uint16
|
| 379 |
+
datasets: null
|
| 380 |
+
label_mask_paths: null
|
| 381 |
+
pad_direction: right
|
| 382 |
+
generate_attention_mask: false
|
| 383 |
+
generate_doc_lengths: false
|
| 384 |
+
num_workers: 0
|
| 385 |
+
drop_last: false
|
| 386 |
+
pin_memory: false
|
| 387 |
+
prefetch_factor: null
|
| 388 |
+
persistent_workers: false
|
| 389 |
+
timeout: 0
|
| 390 |
+
seed: null
|
| 391 |
+
instance_filter: null
|
| 392 |
+
device_eval_batch_size: null
|
| 393 |
+
subset_num_batches: null
|
| 394 |
+
- label: social_iqa
|
| 395 |
+
type: downstream
|
| 396 |
+
data:
|
| 397 |
+
paths: null
|
| 398 |
+
memmap_dtype: uint16
|
| 399 |
+
datasets: null
|
| 400 |
+
label_mask_paths: null
|
| 401 |
+
pad_direction: right
|
| 402 |
+
generate_attention_mask: false
|
| 403 |
+
generate_doc_lengths: false
|
| 404 |
+
num_workers: 0
|
| 405 |
+
drop_last: false
|
| 406 |
+
pin_memory: false
|
| 407 |
+
prefetch_factor: null
|
| 408 |
+
persistent_workers: false
|
| 409 |
+
timeout: 0
|
| 410 |
+
seed: null
|
| 411 |
+
instance_filter: null
|
| 412 |
+
device_eval_batch_size: null
|
| 413 |
+
subset_num_batches: null
|
| 414 |
+
- label: mmlu_stem_var
|
| 415 |
+
type: downstream
|
| 416 |
+
data:
|
| 417 |
+
paths: null
|
| 418 |
+
memmap_dtype: uint16
|
| 419 |
+
datasets: null
|
| 420 |
+
label_mask_paths: null
|
| 421 |
+
pad_direction: right
|
| 422 |
+
generate_attention_mask: false
|
| 423 |
+
generate_doc_lengths: false
|
| 424 |
+
num_workers: 0
|
| 425 |
+
drop_last: false
|
| 426 |
+
pin_memory: false
|
| 427 |
+
prefetch_factor: null
|
| 428 |
+
persistent_workers: false
|
| 429 |
+
timeout: 0
|
| 430 |
+
seed: null
|
| 431 |
+
instance_filter: null
|
| 432 |
+
device_eval_batch_size: null
|
| 433 |
+
subset_num_batches: null
|
| 434 |
+
- label: mmlu_humanities_var
|
| 435 |
+
type: downstream
|
| 436 |
+
data:
|
| 437 |
+
paths: null
|
| 438 |
+
memmap_dtype: uint16
|
| 439 |
+
datasets: null
|
| 440 |
+
label_mask_paths: null
|
| 441 |
+
pad_direction: right
|
| 442 |
+
generate_attention_mask: false
|
| 443 |
+
generate_doc_lengths: false
|
| 444 |
+
num_workers: 0
|
| 445 |
+
drop_last: false
|
| 446 |
+
pin_memory: false
|
| 447 |
+
prefetch_factor: null
|
| 448 |
+
persistent_workers: false
|
| 449 |
+
timeout: 0
|
| 450 |
+
seed: null
|
| 451 |
+
instance_filter: null
|
| 452 |
+
device_eval_batch_size: null
|
| 453 |
+
subset_num_batches: null
|
| 454 |
+
- label: mmlu_social_sciences_var
|
| 455 |
+
type: downstream
|
| 456 |
+
data:
|
| 457 |
+
paths: null
|
| 458 |
+
memmap_dtype: uint16
|
| 459 |
+
datasets: null
|
| 460 |
+
label_mask_paths: null
|
| 461 |
+
pad_direction: right
|
| 462 |
+
generate_attention_mask: false
|
| 463 |
+
generate_doc_lengths: false
|
| 464 |
+
num_workers: 0
|
| 465 |
+
drop_last: false
|
| 466 |
+
pin_memory: false
|
| 467 |
+
prefetch_factor: null
|
| 468 |
+
persistent_workers: false
|
| 469 |
+
timeout: 0
|
| 470 |
+
seed: null
|
| 471 |
+
instance_filter: null
|
| 472 |
+
device_eval_batch_size: null
|
| 473 |
+
subset_num_batches: null
|
| 474 |
+
- label: mmlu_other_var
|
| 475 |
+
type: downstream
|
| 476 |
+
data:
|
| 477 |
+
paths: null
|
| 478 |
+
memmap_dtype: uint16
|
| 479 |
+
datasets: null
|
| 480 |
+
label_mask_paths: null
|
| 481 |
+
pad_direction: right
|
| 482 |
+
generate_attention_mask: false
|
| 483 |
+
generate_doc_lengths: false
|
| 484 |
+
num_workers: 0
|
| 485 |
+
drop_last: false
|
| 486 |
+
pin_memory: false
|
| 487 |
+
prefetch_factor: null
|
| 488 |
+
persistent_workers: false
|
| 489 |
+
timeout: 0
|
| 490 |
+
seed: null
|
| 491 |
+
instance_filter: null
|
| 492 |
+
device_eval_batch_size: null
|
| 493 |
+
subset_num_batches: null
|
| 494 |
+
- label: mmlu_stem_mc_5shot
|
| 495 |
+
type: downstream
|
| 496 |
+
data:
|
| 497 |
+
paths: null
|
| 498 |
+
memmap_dtype: uint16
|
| 499 |
+
datasets: null
|
| 500 |
+
label_mask_paths: null
|
| 501 |
+
pad_direction: right
|
| 502 |
+
generate_attention_mask: false
|
| 503 |
+
generate_doc_lengths: false
|
| 504 |
+
num_workers: 0
|
| 505 |
+
drop_last: false
|
| 506 |
+
pin_memory: false
|
| 507 |
+
prefetch_factor: null
|
| 508 |
+
persistent_workers: false
|
| 509 |
+
timeout: 0
|
| 510 |
+
seed: null
|
| 511 |
+
instance_filter: null
|
| 512 |
+
device_eval_batch_size: null
|
| 513 |
+
subset_num_batches: null
|
| 514 |
+
- label: mmlu_humanities_mc_5shot
|
| 515 |
+
type: downstream
|
| 516 |
+
data:
|
| 517 |
+
paths: null
|
| 518 |
+
memmap_dtype: uint16
|
| 519 |
+
datasets: null
|
| 520 |
+
label_mask_paths: null
|
| 521 |
+
pad_direction: right
|
| 522 |
+
generate_attention_mask: false
|
| 523 |
+
generate_doc_lengths: false
|
| 524 |
+
num_workers: 0
|
| 525 |
+
drop_last: false
|
| 526 |
+
pin_memory: false
|
| 527 |
+
prefetch_factor: null
|
| 528 |
+
persistent_workers: false
|
| 529 |
+
timeout: 0
|
| 530 |
+
seed: null
|
| 531 |
+
instance_filter: null
|
| 532 |
+
device_eval_batch_size: null
|
| 533 |
+
subset_num_batches: null
|
| 534 |
+
- label: mmlu_social_sciences_mc_5shot
|
| 535 |
+
type: downstream
|
| 536 |
+
data:
|
| 537 |
+
paths: null
|
| 538 |
+
memmap_dtype: uint16
|
| 539 |
+
datasets: null
|
| 540 |
+
label_mask_paths: null
|
| 541 |
+
pad_direction: right
|
| 542 |
+
generate_attention_mask: false
|
| 543 |
+
generate_doc_lengths: false
|
| 544 |
+
num_workers: 0
|
| 545 |
+
drop_last: false
|
| 546 |
+
pin_memory: false
|
| 547 |
+
prefetch_factor: null
|
| 548 |
+
persistent_workers: false
|
| 549 |
+
timeout: 0
|
| 550 |
+
seed: null
|
| 551 |
+
instance_filter: null
|
| 552 |
+
device_eval_batch_size: null
|
| 553 |
+
subset_num_batches: null
|
| 554 |
+
- label: mmlu_other_mc_5shot
|
| 555 |
+
type: downstream
|
| 556 |
+
data:
|
| 557 |
+
paths: null
|
| 558 |
+
memmap_dtype: uint16
|
| 559 |
+
datasets: null
|
| 560 |
+
label_mask_paths: null
|
| 561 |
+
pad_direction: right
|
| 562 |
+
generate_attention_mask: false
|
| 563 |
+
generate_doc_lengths: false
|
| 564 |
+
num_workers: 0
|
| 565 |
+
drop_last: false
|
| 566 |
+
pin_memory: false
|
| 567 |
+
prefetch_factor: null
|
| 568 |
+
persistent_workers: false
|
| 569 |
+
timeout: 0
|
| 570 |
+
seed: null
|
| 571 |
+
instance_filter: null
|
| 572 |
+
device_eval_batch_size: null
|
| 573 |
+
subset_num_batches: null
|
| 574 |
+
- label: mmlu_stem_mc_5shot_test
|
| 575 |
+
type: downstream
|
| 576 |
+
data:
|
| 577 |
+
paths: null
|
| 578 |
+
memmap_dtype: uint16
|
| 579 |
+
datasets: null
|
| 580 |
+
label_mask_paths: null
|
| 581 |
+
pad_direction: right
|
| 582 |
+
generate_attention_mask: false
|
| 583 |
+
generate_doc_lengths: false
|
| 584 |
+
num_workers: 0
|
| 585 |
+
drop_last: false
|
| 586 |
+
pin_memory: false
|
| 587 |
+
prefetch_factor: null
|
| 588 |
+
persistent_workers: false
|
| 589 |
+
timeout: 0
|
| 590 |
+
seed: null
|
| 591 |
+
instance_filter: null
|
| 592 |
+
device_eval_batch_size: null
|
| 593 |
+
subset_num_batches: null
|
| 594 |
+
- label: mmlu_humanities_mc_5shot_test
|
| 595 |
+
type: downstream
|
| 596 |
+
data:
|
| 597 |
+
paths: null
|
| 598 |
+
memmap_dtype: uint16
|
| 599 |
+
datasets: null
|
| 600 |
+
label_mask_paths: null
|
| 601 |
+
pad_direction: right
|
| 602 |
+
generate_attention_mask: false
|
| 603 |
+
generate_doc_lengths: false
|
| 604 |
+
num_workers: 0
|
| 605 |
+
drop_last: false
|
| 606 |
+
pin_memory: false
|
| 607 |
+
prefetch_factor: null
|
| 608 |
+
persistent_workers: false
|
| 609 |
+
timeout: 0
|
| 610 |
+
seed: null
|
| 611 |
+
instance_filter: null
|
| 612 |
+
device_eval_batch_size: null
|
| 613 |
+
subset_num_batches: null
|
| 614 |
+
- label: mmlu_social_sciences_mc_5shot_test
|
| 615 |
+
type: downstream
|
| 616 |
+
data:
|
| 617 |
+
paths: null
|
| 618 |
+
memmap_dtype: uint16
|
| 619 |
+
datasets: null
|
| 620 |
+
label_mask_paths: null
|
| 621 |
+
pad_direction: right
|
| 622 |
+
generate_attention_mask: false
|
| 623 |
+
generate_doc_lengths: false
|
| 624 |
+
num_workers: 0
|
| 625 |
+
drop_last: false
|
| 626 |
+
pin_memory: false
|
| 627 |
+
prefetch_factor: null
|
| 628 |
+
persistent_workers: false
|
| 629 |
+
timeout: 0
|
| 630 |
+
seed: null
|
| 631 |
+
instance_filter: null
|
| 632 |
+
device_eval_batch_size: null
|
| 633 |
+
subset_num_batches: null
|
| 634 |
+
- label: mmlu_other_mc_5shot_test
|
| 635 |
+
type: downstream
|
| 636 |
+
data:
|
| 637 |
+
paths: null
|
| 638 |
+
memmap_dtype: uint16
|
| 639 |
+
datasets: null
|
| 640 |
+
label_mask_paths: null
|
| 641 |
+
pad_direction: right
|
| 642 |
+
generate_attention_mask: false
|
| 643 |
+
generate_doc_lengths: false
|
| 644 |
+
num_workers: 0
|
| 645 |
+
drop_last: false
|
| 646 |
+
pin_memory: false
|
| 647 |
+
prefetch_factor: null
|
| 648 |
+
persistent_workers: false
|
| 649 |
+
timeout: 0
|
| 650 |
+
seed: null
|
| 651 |
+
instance_filter: null
|
| 652 |
+
device_eval_batch_size: null
|
| 653 |
+
subset_num_batches: null
|
| 654 |
+
eval_interval: 2000
|
| 655 |
+
tokenizer:
|
| 656 |
+
identifier: tokenizers/allenai_gpt-neox-olmo-dolma-v1_5.json
|
| 657 |
+
truncate_direction: right
|
| 658 |
+
save_folder: runs/olmoe-7b-29b-2ep
|
| 659 |
+
remote_save_folder: null
|
| 660 |
+
canceled_check_interval: 50
|
| 661 |
+
save_interval: 2000
|
| 662 |
+
save_interval_unsharded: null
|
| 663 |
+
save_interval_ephemeral: null
|
| 664 |
+
save_num_checkpoints_to_keep: 5
|
| 665 |
+
save_num_unsharded_checkpoints_to_keep: -1
|
| 666 |
+
save_overwrite: true
|
| 667 |
+
force_save_unsharded: false
|
| 668 |
+
no_pre_train_checkpoint: true
|
| 669 |
+
load_path: null
|
| 670 |
+
load_path_sharded_checkpointer: null
|
| 671 |
+
try_load_latest_save: false
|
| 672 |
+
reset_optimizer_state: false
|
| 673 |
+
reset_trainer_state: false
|
| 674 |
+
sharded_checkpointer: olmo_core
|
| 675 |
+
new_style_checkpoints: null
|
| 676 |
+
max_duration: 2ep
|
| 677 |
+
global_train_batch_size: 1024
|
| 678 |
+
device_train_batch_size: 256
|
| 679 |
+
device_train_microbatch_size: 2
|
| 680 |
+
device_eval_batch_size: 1
|
| 681 |
+
eval_subset_num_batches: -1
|
| 682 |
+
eval_on_load: false
|
| 683 |
+
device_train_grad_accum: 128
|
| 684 |
+
max_grad_norm: 1.0
|
| 685 |
+
max_grad_norm_ratio: null
|
| 686 |
+
precision: amp_bf16
|
| 687 |
+
wandb:
|
| 688 |
+
project: hiomoe
|
| 689 |
+
entity: bulgakov
|
| 690 |
+
group: 500gb-test
|
| 691 |
+
name: olmoe-7b-29b-2ep
|
| 692 |
+
tags:
|
| 693 |
+
- watching
|
| 694 |
+
log_artifacts: false
|
| 695 |
+
rank_zero_only: true
|
| 696 |
+
log_interval: 1
|
| 697 |
+
speed_monitor:
|
| 698 |
+
window_size: 1
|
| 699 |
+
gpu_flops_available: null
|
| 700 |
+
console_log_interval: 1
|
| 701 |
+
gen1_gc_interval: 1
|
| 702 |
+
compile: null
|
| 703 |
+
distributed_strategy: fsdp
|
| 704 |
+
fsdp:
|
| 705 |
+
use_orig_params: true
|
| 706 |
+
sharding_strategy: FULL_SHARD
|
| 707 |
+
wrapping_strategy: by_block
|
| 708 |
+
precision: mixed
|
| 709 |
+
hybrid_sharding_num_model_replicas: null
|
| 710 |
+
ddp: null
|
| 711 |
+
softmax_auxiliary_loss: false
|
| 712 |
+
auxiliary_loss_multiplier: 0.0001
|
| 713 |
+
time_limit: null
|
| 714 |
+
extra_steps_after_cancel: 10
|
| 715 |
+
early_stopping_factor: null
|
| 716 |
+
save_data_indices: true
|
| 717 |
+
python_profiling: false
|
| 718 |
+
torch_profiling: false
|
| 719 |
+
stop_at: 13964
|
| 720 |
+
stop_after: null
|
| 721 |
+
activation_checkpointing: fine_grained
|
| 722 |
+
fused_loss: null
|
| 723 |
+
hf_datasets_cache_dir: null
|
| 724 |
+
module_outputs_save_steps: null
|
step8000/model/metadata.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"tensors": {"transformer.wte.weight": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 51511808]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[51511808, 103022592]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.ln_f.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 1024]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[1024, 2048]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.ff_out.weight": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 51510784]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[51510784, 103022592]], "local_shape": null, "global_offset": null}}}}}
|
step8000/optim/metadata.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
step8000/train/rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:55ce849fb5cf4a8f13e62bbe6888f4296d535f18938efc7de53bd7b7e4c3d84d
|
| 3 |
+
size 14997
|
step8000/train/rank1.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4c6e8b4c1f4079dd71c5cd3c8b8912f7023501542fdfefb7db5d360bbdbbf6dc
|
| 3 |
+
size 14997
|
step8000/train/rank2.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:abecbc4654726d4e171f18c7c70c809e8e61a87979e8d1adc88fde466ab9050b
|
| 3 |
+
size 14997
|
step8000/train/rank3.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9485998d23382706ca787f8ef110b0ac28ad142a8429f9efbb4b5c12dc2a774c
|
| 3 |
+
size 14997
|
train_data/global_indices.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:722c0fcfa8661ed55afcc890b069ffcfbd4c0e80cf26b3a8f282536bbe3cfd19
|
| 3 |
+
size 28581312
|
wandb/wandb/debug-internal.log
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2025-12-19T03:28:09.715068381Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
|
| 2 |
+
{"time":"2025-12-19T03:28:10.010671632Z","level":"INFO","msg":"stream: created new stream","id":"fhag09gx"}
|
| 3 |
+
{"time":"2025-12-19T03:28:10.01079177Z","level":"INFO","msg":"writer: started","stream_id":"fhag09gx"}
|
| 4 |
+
{"time":"2025-12-19T03:28:10.010792247Z","level":"INFO","msg":"sender: started","stream_id":"fhag09gx"}
|
| 5 |
+
{"time":"2025-12-19T03:28:10.011088687Z","level":"INFO","msg":"handler: started","stream_id":"fhag09gx"}
|
| 6 |
+
{"time":"2025-12-19T03:28:10.011242775Z","level":"INFO","msg":"stream: started","id":"fhag09gx"}
|
| 7 |
+
{"time":"2025-12-19T03:28:10.069884068Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
|
| 8 |
+
{"time":"2025-12-25T13:58:15.121256706Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
|
| 9 |
+
{"time":"2025-12-25T13:58:15.388114558Z","level":"INFO","msg":"handler: operation stats","stats":{}}
|
| 10 |
+
{"time":"2025-12-25T13:58:15.404292716Z","level":"INFO","msg":"stream: closing","id":"fhag09gx"}
|
| 11 |
+
{"time":"2025-12-25T13:58:15.404311317Z","level":"INFO","msg":"handler: closed","stream_id":"fhag09gx"}
|
| 12 |
+
{"time":"2025-12-25T13:58:15.404690567Z","level":"INFO","msg":"sender: closed","stream_id":"fhag09gx"}
|
| 13 |
+
{"time":"2025-12-25T13:58:15.404702005Z","level":"INFO","msg":"stream: closed","id":"fhag09gx"}
|
wandb/wandb/debug.log
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2025-12-19 03:28:09,456 INFO MainThread:1675715 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
|
| 2 |
+
2025-12-19 03:28:09,456 INFO MainThread:1675715 [wandb_setup.py:_flush():80] Configure stats pid to 1675715
|
| 3 |
+
2025-12-19 03:28:09,456 INFO MainThread:1675715 [wandb_setup.py:_flush():80] Loading settings from /data/users/xandi281/.config/wandb/settings
|
| 4 |
+
2025-12-19 03:28:09,456 INFO MainThread:1675715 [wandb_setup.py:_flush():80] Loading settings from /data/users/xandi281/work/Hi-OLMoE/OLMo/wandb/settings
|
| 5 |
+
2025-12-19 03:28:09,456 INFO MainThread:1675715 [wandb_setup.py:_flush():80] Loading settings from environment variables
|
| 6 |
+
2025-12-19 03:28:09,456 INFO MainThread:1675715 [wandb_init.py:setup_run_log_directory():714] Logging user logs to runs/olmoe-7b-29b-2ep/wandb/wandb/run-20251219_032809-fhag09gx/logs/debug.log
|
| 7 |
+
2025-12-19 03:28:09,456 INFO MainThread:1675715 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to runs/olmoe-7b-29b-2ep/wandb/wandb/run-20251219_032809-fhag09gx/logs/debug-internal.log
|
| 8 |
+
2025-12-19 03:28:09,456 INFO MainThread:1675715 [wandb_init.py:init():841] calling init triggers
|
| 9 |
+
2025-12-19 03:28:09,457 INFO MainThread:1675715 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
|
| 10 |
+
config: {'run_name': 'olmoe-7b-29b-2ep', 'seed': 6198, 'epoch': None, 'dry_run': False, 'model': {'d_model': 2048, 'n_heads': 16, 'n_kv_heads': None, 'clip_qkv': None, 'n_layers': 16, 'mlp_ratio': 1, 'mlp_hidden_size': None, 'activation_type': 'swiglu', 'block_type': 'moe', 'block_group_size': 1, 'alibi': False, 'alibi_bias_max': 8.0, 'rope': True, 'rope_full_precision': True, 'rope_theta': 10000, 'flash_attention': True, 'attention_dropout': 0.0, 'multi_query_attention': None, 'attention_layer_norm': True, 'residual_dropout': 0.0, 'embedding_dropout': 0.0, 'embedding_layer_norm': False, 'layer_norm_type': 'rms', 'layer_norm_with_affine': True, 'layer_norm_eps': 1e-05, 'attention_layer_norm_with_affine': True, 'max_sequence_length': 4096, 'include_bias': False, 'bias_for_layer_norm': False, 'scale_logits': False, 'vocab_size': 50280, 'embedding_size': 50304, 'weight_tying': False, 'eos_token_id': 0, 'pad_token_id': 1, 'init_device': 'meta', 'init_fn': 'normal', 'init_std': 0.02, 'init_cutoff_factor': 3.0, 'precision': 'amp_bf16', 'moe_num_experts': 64, 'moe_top_k': 8, 'moe_mlp_impl': 'sparse', 'moe_log_expert_assignment': False, 'moe_shared_expert': False, 'moe_lbl_in_fp32': False, 'moe_loss_weight': 0.01, 'moe_zloss_weight': 0.001, 'moe_dropless': False, 'moe_capacity_factor': 1.25, 'himoe_enabled': False, 'himoe_n_groups': 4, 'himoe_top_k_per_group': 2, 'himoe_lambda1': 0.01, 'himoe_lambda2': 0.1, 'scale_emb_init': False, 'emb_init_std': None, 'norm_after': False}, 'optimizer': {'name': 'adamw', 'learning_rate': 0.0004, 'weight_decay': 0.1, 'betas': (0.9, 0.95), 'eps': 1e-08, 'no_decay_norm_and_bias': None, 'selective_updates': False, 'decay_norm_and_bias': True, 'decay_embeddings': True, 'metrics_log_interval': 10, 'record_update_metrics': False}, 'scheduler': {'name': 'cosine_with_warmup', 'units': 'tokens', 't_warmup': 1405296000, 't_max': 58540000000, 'alpha_f': 0.1, 'grad_clip_warmup_steps': None, 'grad_clip_warmup_factor': None, 'warmup_min_lr': None}, 'data': {'paths': ['/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-13-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-14-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-15-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-000-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-001-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-002-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-003-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-004-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-005-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-006-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-007-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-008-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-009-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-010-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-011-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-012-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-013-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy'], 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 8, 'drop_last': True, 'pin_memory': True, 'prefetch_factor': 8, 'persistent_workers': True, 'timeout': 0, 'seed': None, 'instance_filter': {'repetition_max_period': 13, 'repetition_min_period': 1, 'repetition_max_count': 32}}, 'restore_dataloader': True, 'fast_forward_batches': None, 'evaluators': [{'label': 'all-small-ppl-validation', 'type': 'lm', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': {'c4_en-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/c4_en/val/part-0-00000.npy'], 'dolma_books-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_books/val/part-0-00000.npy'], 'dolma_common-crawl-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_common-crawl/val/part-0-00000.npy'], 'dolma_pes2o-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_pes2o/val/part-0-00000.npy'], 'dolma_reddit-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_reddit/val/part-0-00000.npy'], 'dolma_stack-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_stack/val/part-0-00000.npy'], 'dolma_wiki-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_wiki/val/part-0-00000.npy'], 'ice-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/ice/val/part-0-00000.npy'], 'm2d2_s2orc-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/m2d2_s2orc/val/part-0-00000.npy'], 'pile-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/pile/val/part-0-00000.npy'], 'wikitext_103-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/wikitext_103/val/part-0-00000.npy']}, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': True, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'piqa', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'hellaswag', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'winogrande', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'openbook_qa', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'boolq', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'sciq', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'arc_easy', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'arc_challenge', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'copa', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'commonsense_qa', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'social_iqa', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_stem_var', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_humanities_var', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_social_sciences_var', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_other_var', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_stem_mc_5shot', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_humanities_mc_5shot', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_social_sciences_mc_5shot', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_other_mc_5shot', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_stem_mc_5shot_test', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_humanities_mc_5shot_test', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_social_sciences_mc_5shot_test', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_other_mc_5shot_test', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}], 'eval_interval': 2000, 'tokenizer': {'identifier': 'tokenizers/allenai_gpt-neox-olmo-dolma-v1_5.json', 'truncate_direction': 'right'}, 'save_folder': 'runs/olmoe-7b-29b-2ep', 'remote_save_folder': None, 'canceled_check_interval': 50, 'save_interval': 2000, 'save_interval_unsharded': None, 'save_interval_ephemeral': None, 'save_num_checkpoints_to_keep': 5, 'save_num_unsharded_checkpoints_to_keep': -1, 'save_overwrite': True, 'force_save_unsharded': False, 'no_pre_train_checkpoint': True, 'load_path': None, 'load_path_sharded_checkpointer': None, 'try_load_latest_save': False, 'reset_optimizer_state': False, 'reset_trainer_state': False, 'sharded_checkpointer': 'olmo_core', 'new_style_checkpoints': None, 'max_duration': '2ep', 'global_train_batch_size': 1024, 'device_train_batch_size': 256, 'device_train_microbatch_size': 2, 'device_eval_batch_size': 1, 'eval_subset_num_batches': -1, 'eval_on_load': False, 'device_train_grad_accum': 128, 'max_grad_norm': 1.0, 'max_grad_norm_ratio': None, 'precision': 'amp_bf16', 'speed_monitor': {'window_size': 1, 'gpu_flops_available': None}, 'console_log_interval': 1, 'gen1_gc_interval': 1, 'compile': None, 'distributed_strategy': 'fsdp', 'fsdp': {'use_orig_params': True, 'sharding_strategy': <ShardingStrategy.FULL_SHARD: 1>, 'wrapping_strategy': 'by_block', 'precision': 'mixed', 'hybrid_sharding_num_model_replicas': None}, 'ddp': None, 'softmax_auxiliary_loss': False, 'auxiliary_loss_multiplier': 0.0001, 'time_limit': None, 'extra_steps_after_cancel': 10, 'early_stopping_factor': None, 'save_data_indices': True, 'python_profiling': False, 'torch_profiling': False, 'stop_at': None, 'stop_after': None, 'activation_checkpointing': 'fine_grained', 'fused_loss': None, 'hf_datasets_cache_dir': None, 'module_outputs_save_steps': None, '_wandb': {'code_path': 'code/OLMo/scripts/train.py'}}
|
| 11 |
+
2025-12-19 03:28:09,457 INFO MainThread:1675715 [wandb_init.py:init():889] starting backend
|
| 12 |
+
2025-12-19 03:28:09,710 INFO MainThread:1675715 [wandb_init.py:init():892] sending inform_init request
|
| 13 |
+
2025-12-19 03:28:09,713 INFO MainThread:1675715 [wandb_init.py:init():900] backend started and connected
|
| 14 |
+
2025-12-19 03:28:09,716 INFO MainThread:1675715 [wandb_init.py:init():970] updated telemetry
|
| 15 |
+
2025-12-19 03:28:09,718 INFO MainThread:1675715 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
|
| 16 |
+
2025-12-19 03:28:10,068 INFO MainThread:1675715 [wandb_init.py:init():1041] starting run threads in backend
|
| 17 |
+
2025-12-19 03:28:10,155 INFO MainThread:1675715 [wandb_run.py:_console_start():2521] atexit reg
|
| 18 |
+
2025-12-19 03:28:10,155 INFO MainThread:1675715 [wandb_run.py:_redirect():2369] redirect: wrap_raw
|
| 19 |
+
2025-12-19 03:28:10,155 INFO MainThread:1675715 [wandb_run.py:_redirect():2438] Wrapping output streams.
|
| 20 |
+
2025-12-19 03:28:10,155 INFO MainThread:1675715 [wandb_run.py:_redirect():2461] Redirects installed.
|
| 21 |
+
2025-12-19 03:28:10,158 INFO MainThread:1675715 [wandb_init.py:init():1081] run started, returning control to user process
|
| 22 |
+
2025-12-25 13:58:11,885 INFO MainThread:1675715 [wandb_run.py:_finish():2287] finishing run bulgakov/hiomoe/fhag09gx
|
| 23 |
+
2025-12-25 13:58:11,885 INFO MainThread:1675715 [wandb_run.py:_atexit_cleanup():2486] got exitcode: 0
|
| 24 |
+
2025-12-25 13:58:11,885 INFO MainThread:1675715 [wandb_run.py:_restore():2468] restore
|
| 25 |
+
2025-12-25 13:58:11,885 INFO MainThread:1675715 [wandb_run.py:_restore():2474] restore done
|
| 26 |
+
2025-12-25 13:58:15,403 INFO MainThread:1675715 [wandb_run.py:_footer_sync_info():3862] logging synced files
|
wandb/wandb/run-20251219_032809-fhag09gx/files/code/OLMo/scripts/train.py
ADDED
|
@@ -0,0 +1,373 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Run this script with 'torchrun'."""
|
| 2 |
+
|
| 3 |
+
import gzip
|
| 4 |
+
import logging
|
| 5 |
+
import sys
|
| 6 |
+
from datetime import timedelta
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
from typing import Optional, TextIO
|
| 9 |
+
|
| 10 |
+
import torch
|
| 11 |
+
import torch.distributed as dist
|
| 12 |
+
import torch.multiprocessing as mp
|
| 13 |
+
import wandb
|
| 14 |
+
from packaging import version
|
| 15 |
+
from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
|
| 16 |
+
from torch.distributed.fsdp import ShardingStrategy
|
| 17 |
+
from torch.nn.parallel import DistributedDataParallel as DDP
|
| 18 |
+
|
| 19 |
+
from olmo.config import (
|
| 20 |
+
ActivationCheckpointingStrategy,
|
| 21 |
+
CheckpointType,
|
| 22 |
+
DDPGradSyncMode,
|
| 23 |
+
DistributedStrategy,
|
| 24 |
+
TrainConfig,
|
| 25 |
+
)
|
| 26 |
+
from olmo.data import build_train_dataloader
|
| 27 |
+
from olmo.eval import build_evaluators
|
| 28 |
+
from olmo.exceptions import OLMoCliError, OLMoConfigurationError
|
| 29 |
+
from olmo.model import OLMo
|
| 30 |
+
from olmo.optim import BoltOnWarmupScheduler, build_optimizer, build_scheduler
|
| 31 |
+
from olmo.torch_util import (
|
| 32 |
+
barrier,
|
| 33 |
+
get_default_device,
|
| 34 |
+
get_global_rank,
|
| 35 |
+
get_local_rank,
|
| 36 |
+
get_local_world_size,
|
| 37 |
+
get_world_size,
|
| 38 |
+
peak_gpu_memory,
|
| 39 |
+
seed_all,
|
| 40 |
+
)
|
| 41 |
+
from olmo.train import Trainer
|
| 42 |
+
from olmo.util import (
|
| 43 |
+
add_cached_path_clients,
|
| 44 |
+
clean_opt,
|
| 45 |
+
find_latest_checkpoint,
|
| 46 |
+
log_extra_field,
|
| 47 |
+
prepare_cli_environment,
|
| 48 |
+
)
|
| 49 |
+
|
| 50 |
+
log = logging.getLogger("train")
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def main(cfg: TrainConfig) -> None:
|
| 54 |
+
# Ensure run name set.
|
| 55 |
+
if cfg.run_name is None:
|
| 56 |
+
raise OLMoConfigurationError("--run_name is required")
|
| 57 |
+
log_extra_field("run_name", cfg.run_name)
|
| 58 |
+
|
| 59 |
+
# Sanity check
|
| 60 |
+
if (cfg.reset_optimizer_state or cfg.reset_trainer_state) and cfg.load_path is None:
|
| 61 |
+
log.warning(
|
| 62 |
+
"You want to reset the optimizer or trainer state, but we're not loading from the checkpoint. The"
|
| 63 |
+
"setting has no effect."
|
| 64 |
+
)
|
| 65 |
+
|
| 66 |
+
barrier()
|
| 67 |
+
|
| 68 |
+
device = torch.device("cuda")
|
| 69 |
+
|
| 70 |
+
# Fill some configuration options.
|
| 71 |
+
cfg.model.precision = cfg.precision
|
| 72 |
+
cfg.device_train_batch_size = cfg.global_train_batch_size // get_world_size()
|
| 73 |
+
assert cfg.device_train_batch_size is not None # for mypy
|
| 74 |
+
cfg.device_train_grad_accum = cfg.device_train_batch_size // cfg.device_train_microbatch_size
|
| 75 |
+
if cfg.optimizer.no_decay_norm_and_bias is not None:
|
| 76 |
+
log.warning(
|
| 77 |
+
"You set the deprecated config option `no_decay_norm_and_bias`. For compatibility, this"
|
| 78 |
+
"setting will take precedence over all other weight decay configurations. Please change"
|
| 79 |
+
"your config to use `decay_norm_and_bias` and `decay_embeddings` instead."
|
| 80 |
+
)
|
| 81 |
+
cfg.optimizer.decay_norm_and_bias = not cfg.optimizer.no_decay_norm_and_bias
|
| 82 |
+
cfg.optimizer.decay_embeddings = not cfg.optimizer.no_decay_norm_and_bias
|
| 83 |
+
cfg.optimizer.no_decay_norm_and_bias = None # So nobody uses this by accident.
|
| 84 |
+
|
| 85 |
+
# Display and save configuration.
|
| 86 |
+
if get_global_rank() == 0:
|
| 87 |
+
if cfg.data.paths is not None and len(cfg.data.paths) < 50:
|
| 88 |
+
log.info("Configuration:")
|
| 89 |
+
log.info(cfg)
|
| 90 |
+
if not cfg.dry_run and (cfg.load_path is None or Path(cfg.load_path).parent != Path(cfg.save_folder)):
|
| 91 |
+
# Save config.
|
| 92 |
+
save_path = Path(cfg.save_folder) / "config.yaml"
|
| 93 |
+
if save_path.is_file() and not cfg.save_overwrite:
|
| 94 |
+
raise OLMoConfigurationError(f"{save_path} already exists, use --save_overwrite to overwrite")
|
| 95 |
+
else:
|
| 96 |
+
log.info(f"Saving config to {save_path}")
|
| 97 |
+
save_path.parent.mkdir(exist_ok=True, parents=True)
|
| 98 |
+
cfg.save(save_path)
|
| 99 |
+
del save_path
|
| 100 |
+
|
| 101 |
+
barrier()
|
| 102 |
+
|
| 103 |
+
# Maybe start W&B run.
|
| 104 |
+
if cfg.wandb is not None and (get_global_rank() == 0 or not cfg.wandb.rank_zero_only):
|
| 105 |
+
wandb_dir = Path(cfg.save_folder) / "wandb"
|
| 106 |
+
wandb_dir.mkdir(parents=True, exist_ok=True)
|
| 107 |
+
wandb.init(
|
| 108 |
+
dir=wandb_dir,
|
| 109 |
+
project=cfg.wandb.project,
|
| 110 |
+
entity=cfg.wandb.entity,
|
| 111 |
+
group=cfg.wandb.group,
|
| 112 |
+
name=cfg.wandb.name,
|
| 113 |
+
tags=cfg.wandb.tags,
|
| 114 |
+
config=cfg.asdict(exclude=["wandb"]),
|
| 115 |
+
)
|
| 116 |
+
|
| 117 |
+
barrier()
|
| 118 |
+
|
| 119 |
+
# Set seed.
|
| 120 |
+
seed_all(cfg.seed)
|
| 121 |
+
|
| 122 |
+
# Construct data loader.
|
| 123 |
+
train_loader = build_train_dataloader(cfg)
|
| 124 |
+
|
| 125 |
+
# Construct evaluators.
|
| 126 |
+
evaluators = build_evaluators(cfg, device)
|
| 127 |
+
barrier()
|
| 128 |
+
|
| 129 |
+
# Initialize the model.
|
| 130 |
+
log.info("Building model...")
|
| 131 |
+
if (
|
| 132 |
+
cfg.model.block_type == "moe"
|
| 133 |
+
and cfg.activation_checkpointing
|
| 134 |
+
and cfg.activation_checkpointing != ActivationCheckpointingStrategy.fine_grained
|
| 135 |
+
):
|
| 136 |
+
raise OLMoConfigurationError(
|
| 137 |
+
"Only no or fine-grained activation checkpointing is supported for MoE models."
|
| 138 |
+
)
|
| 139 |
+
|
| 140 |
+
olmo_model = OLMo(cfg.model)
|
| 141 |
+
log.info(f"Total number of parameters: {olmo_model.num_params():,d}")
|
| 142 |
+
log.info(f"Number of non-embedding parameters: {olmo_model.num_params(include_embedding=False):,d}")
|
| 143 |
+
if olmo_model.config.block_type == "moe":
|
| 144 |
+
log.info(f"Number of active parameters: {olmo_model.num_params(include_inactive_params=False):,d}")
|
| 145 |
+
log.info(f"Peak GPU Memory (MB) before {cfg.distributed_strategy}: {int(peak_gpu_memory() or 0)}")
|
| 146 |
+
|
| 147 |
+
olmo_model.set_activation_checkpointing(cfg.activation_checkpointing)
|
| 148 |
+
|
| 149 |
+
if cfg.distributed_strategy == DistributedStrategy.ddp:
|
| 150 |
+
log.info("Wrapping model with DDP...")
|
| 151 |
+
assert cfg.ddp is not None, "DistributedStrategy ddp needs cfg.ddp to be set!"
|
| 152 |
+
|
| 153 |
+
if cfg.model.init_device != "cuda":
|
| 154 |
+
raise OLMoConfigurationError("DDP does not work with init_device set to anything other than `cuda`.")
|
| 155 |
+
|
| 156 |
+
if cfg.ddp.find_unused_params is True and cfg.ddp.grad_sync_mode != DDPGradSyncMode.micro_batch:
|
| 157 |
+
raise OLMoConfigurationError(
|
| 158 |
+
"`find_unused_params` is set to True. DDP needs to synchronize gradients for every micro-batch to avoid errors. Set `grad_sync_mode` to `micro_batch`."
|
| 159 |
+
)
|
| 160 |
+
|
| 161 |
+
param_init_fn = None
|
| 162 |
+
|
| 163 |
+
# move to cuda before calling ddp
|
| 164 |
+
dist_model = DDP(olmo_model.to(device), find_unused_parameters=cfg.ddp.find_unused_params)
|
| 165 |
+
elif cfg.distributed_strategy == DistributedStrategy.fsdp:
|
| 166 |
+
# Wrap the model in FSDP.
|
| 167 |
+
log.info("Wrapping model with FSDP...")
|
| 168 |
+
assert cfg.fsdp is not None, "DistributedStrategy fsdp needs cfg.fsdp to be set!"
|
| 169 |
+
wrap_policy = olmo_model.get_fsdp_wrap_policy(cfg.fsdp.wrapping_strategy)
|
| 170 |
+
|
| 171 |
+
if version.parse(torch.__version__) >= version.parse("2.1.0"):
|
| 172 |
+
# This prevents any parameters from being initialized twice
|
| 173 |
+
def dummy_init_fn(module: torch.nn.Module) -> None:
|
| 174 |
+
module.to_empty(device=get_default_device())
|
| 175 |
+
|
| 176 |
+
param_init_fn = dummy_init_fn
|
| 177 |
+
else:
|
| 178 |
+
param_init_fn = None
|
| 179 |
+
|
| 180 |
+
# Set up device mesh for hybrid sharding in order to specify which nodes are assoicated to a given model replica
|
| 181 |
+
device_mesh = None
|
| 182 |
+
hybrid_sharding_fsdp_kwargs = {}
|
| 183 |
+
if cfg.fsdp.sharding_strategy in (ShardingStrategy.HYBRID_SHARD, ShardingStrategy._HYBRID_SHARD_ZERO2):
|
| 184 |
+
if version.parse(torch.__version__) < version.parse("2.2.0"):
|
| 185 |
+
# Device mesh was not added to PyTorch until v2.2.0
|
| 186 |
+
raise OLMoConfigurationError(
|
| 187 |
+
"OLMo training does not correctly support hybrid sharding before torch 2.2.0"
|
| 188 |
+
)
|
| 189 |
+
|
| 190 |
+
from torch.distributed.device_mesh import init_device_mesh
|
| 191 |
+
|
| 192 |
+
num_model_replicas = cfg.fsdp.hybrid_sharding_num_model_replicas or (
|
| 193 |
+
get_world_size() // get_local_world_size()
|
| 194 |
+
)
|
| 195 |
+
|
| 196 |
+
if num_model_replicas <= 0:
|
| 197 |
+
raise OLMoConfigurationError("fsdp.hybrid_sharding_num_model_replicas must be a positive integer")
|
| 198 |
+
|
| 199 |
+
if get_world_size() % num_model_replicas != 0:
|
| 200 |
+
raise OLMoConfigurationError("fsdp.hybrid_sharding_num_model_replicas must divide world size")
|
| 201 |
+
|
| 202 |
+
device_mesh = init_device_mesh("cuda", (num_model_replicas, get_world_size() // num_model_replicas))
|
| 203 |
+
hybrid_sharding_fsdp_kwargs["device_mesh"] = device_mesh
|
| 204 |
+
|
| 205 |
+
dist_model = FSDP(
|
| 206 |
+
olmo_model,
|
| 207 |
+
sharding_strategy=cfg.fsdp.sharding_strategy,
|
| 208 |
+
mixed_precision=cfg.fsdp_precision,
|
| 209 |
+
auto_wrap_policy=wrap_policy,
|
| 210 |
+
use_orig_params=cfg.fsdp.use_orig_params, # needed for compile and some of our optimizer/parameter metrics
|
| 211 |
+
limit_all_gathers=True,
|
| 212 |
+
device_id=get_local_rank(),
|
| 213 |
+
param_init_fn=param_init_fn,
|
| 214 |
+
**hybrid_sharding_fsdp_kwargs,
|
| 215 |
+
)
|
| 216 |
+
elif cfg.distributed_strategy is None:
|
| 217 |
+
raise NotImplementedError("Single accelerator training not implemented yet!")
|
| 218 |
+
|
| 219 |
+
# when param_init_fn is None, FSDP will call reset_parameters() automatically
|
| 220 |
+
if param_init_fn is not None or cfg.distributed_strategy == DistributedStrategy.ddp:
|
| 221 |
+
olmo_model.reset_parameters()
|
| 222 |
+
|
| 223 |
+
log.info(f"Peak GPU Memory (MB) after {cfg.distributed_strategy}: {int(peak_gpu_memory() or 0)}")
|
| 224 |
+
log.info("Model:")
|
| 225 |
+
log.info(dist_model)
|
| 226 |
+
|
| 227 |
+
# Construct optimizer and learning rate scheduler.
|
| 228 |
+
optim = build_optimizer(cfg, dist_model)
|
| 229 |
+
scheduler = build_scheduler(cfg)
|
| 230 |
+
|
| 231 |
+
# Data indices file.
|
| 232 |
+
indices_file: Optional[TextIO] = None
|
| 233 |
+
if cfg.save_data_indices:
|
| 234 |
+
indices_file_path = Path(cfg.save_folder) / f"data-indices/rank{get_global_rank()}.tsv.gz"
|
| 235 |
+
if indices_file_path.exists() and not cfg.save_overwrite:
|
| 236 |
+
raise OLMoConfigurationError(f"{indices_file_path} already exists, use --save_overwrite to overwrite")
|
| 237 |
+
indices_file_path.parent.mkdir(exist_ok=True, parents=True)
|
| 238 |
+
indices_file = gzip.open(indices_file_path, "wt")
|
| 239 |
+
|
| 240 |
+
# Consolidate components into `Trainer` object.
|
| 241 |
+
with Trainer(
|
| 242 |
+
cfg=cfg,
|
| 243 |
+
epoch=cfg.epoch,
|
| 244 |
+
model=olmo_model,
|
| 245 |
+
dist_model=dist_model,
|
| 246 |
+
optim=optim,
|
| 247 |
+
scheduler=scheduler,
|
| 248 |
+
train_loader=train_loader,
|
| 249 |
+
device=device,
|
| 250 |
+
evaluators=evaluators,
|
| 251 |
+
indices_file=indices_file,
|
| 252 |
+
) as trainer:
|
| 253 |
+
if cfg.try_load_latest_save:
|
| 254 |
+
if (
|
| 255 |
+
cfg.save_folder is not None
|
| 256 |
+
and (checkpoint_dir := find_latest_checkpoint(cfg.save_folder)) is not None
|
| 257 |
+
):
|
| 258 |
+
log.info("Setting load path to local checkpoint %s", checkpoint_dir)
|
| 259 |
+
cfg.load_path = str(checkpoint_dir)
|
| 260 |
+
elif (
|
| 261 |
+
cfg.remote_save_folder is not None
|
| 262 |
+
and (checkpoint_dir := find_latest_checkpoint(cfg.remote_save_folder)) is not None
|
| 263 |
+
):
|
| 264 |
+
log.info("Setting load path to remote checkpoint %s", checkpoint_dir)
|
| 265 |
+
cfg.load_path = str(checkpoint_dir)
|
| 266 |
+
|
| 267 |
+
if not cfg.dry_run and not cfg.no_pre_train_checkpoint and cfg.load_path is None:
|
| 268 |
+
if cfg.distributed_strategy == DistributedStrategy.ddp:
|
| 269 |
+
checkpoint_type = CheckpointType.unsharded
|
| 270 |
+
|
| 271 |
+
if cfg.save_interval_unsharded is None:
|
| 272 |
+
log.warning(
|
| 273 |
+
"DDP requires setting `save_interval_unsharded`. Using the value set for `save_interval`."
|
| 274 |
+
)
|
| 275 |
+
cfg.save_interval_unsharded = cfg.save_interval
|
| 276 |
+
|
| 277 |
+
if cfg.save_num_unsharded_checkpoints_to_keep == 0:
|
| 278 |
+
log.warning(
|
| 279 |
+
"DDP requires setting `save_num_unsharded_checkpoints_to_keep`. Using the value set for `save_num_checkpoints_to_keep`."
|
| 280 |
+
)
|
| 281 |
+
cfg.save_num_unsharded_checkpoints_to_keep = cfg.save_num_checkpoints_to_keep
|
| 282 |
+
elif cfg.distributed_strategy == DistributedStrategy.fsdp:
|
| 283 |
+
checkpoint_type = (
|
| 284 |
+
CheckpointType.sharded if cfg.save_num_checkpoints_to_keep != 0 else CheckpointType.unsharded
|
| 285 |
+
)
|
| 286 |
+
else:
|
| 287 |
+
raise NotImplementedError(f"Distributed strategy {cfg.distributed_strategy} not supported yet!")
|
| 288 |
+
|
| 289 |
+
# We save a checkpoint up-front to make sure this won't fail (due to disk space or whatever).
|
| 290 |
+
log.info("Saving pre-train checkpoint...")
|
| 291 |
+
checkpoint_path, local_checkpoint_cache = trainer.save_checkpoint(checkpoint_type=checkpoint_type)
|
| 292 |
+
log.info(f"Checkpoint saved to {checkpoint_path}")
|
| 293 |
+
|
| 294 |
+
# And they we verify that we can load it.
|
| 295 |
+
log.info("Attempting to load pre-train checkpoint...")
|
| 296 |
+
trainer.restore_checkpoint(
|
| 297 |
+
checkpoint_path, checkpoint_type=checkpoint_type, local_cache=local_checkpoint_cache
|
| 298 |
+
)
|
| 299 |
+
log.info("Checkpoint successfully loaded")
|
| 300 |
+
|
| 301 |
+
# NOTE: https://github.com/allenai/LLM/issues/233
|
| 302 |
+
# log.info("Removing pre-train checkpoint...")
|
| 303 |
+
# trainer.remove_checkpoint(checkpoint_type=checkpoint_type)
|
| 304 |
+
# log.info("Successfully removed checkpoint")
|
| 305 |
+
|
| 306 |
+
if cfg.load_path is not None:
|
| 307 |
+
log.info(f"Loading checkpoint from {cfg.load_path}...")
|
| 308 |
+
trainer.restore_checkpoint(
|
| 309 |
+
cfg.load_path,
|
| 310 |
+
load_optimizer_state=not cfg.reset_optimizer_state,
|
| 311 |
+
load_trainer_state=not cfg.reset_trainer_state,
|
| 312 |
+
sharded_checkpointer=cfg.load_path_sharded_checkpointer,
|
| 313 |
+
)
|
| 314 |
+
log.info("Checkpoint successfully loaded")
|
| 315 |
+
|
| 316 |
+
# If we have to, set a new scheduler:
|
| 317 |
+
if cfg.reset_optimizer_state and not cfg.reset_trainer_state:
|
| 318 |
+
trainer.scheduler = BoltOnWarmupScheduler.wrap(
|
| 319 |
+
trainer.scheduler,
|
| 320 |
+
trainer.global_step,
|
| 321 |
+
int(trainer.global_step + cfg.scheduler.t_warmup),
|
| 322 |
+
)
|
| 323 |
+
|
| 324 |
+
if cfg.force_save_unsharded and cfg.distributed_strategy != DistributedStrategy.ddp:
|
| 325 |
+
log.info("Saving unsharded checkpoint...")
|
| 326 |
+
checkpoint_path, _ = trainer.save_checkpoint(checkpoint_type=CheckpointType.unsharded)
|
| 327 |
+
log.info(f"Unsharded checkpoint saved to {checkpoint_path}")
|
| 328 |
+
|
| 329 |
+
if cfg.compile is not None:
|
| 330 |
+
# TODO (epwalsh): trying to compile the whole train step results in a compile-time error from within
|
| 331 |
+
# the optimizer. We should investigate this further at some point.
|
| 332 |
+
# trainer.train_step = torch.compile(trainer.train_step, **cfg.compile.asdict())
|
| 333 |
+
trainer.train_batch = torch.compile(trainer.train_batch, **cfg.compile.asdict()) # type: ignore
|
| 334 |
+
# TODO (epwalsh): compiling the `eval_batch()` method is a little sketchy since the inputs will look
|
| 335 |
+
# different for different eval tasks. That might be okay, but it might not be.
|
| 336 |
+
# trainer.eval_batch = torch.compile(trainer.eval_batch, **cfg.compile.asdict()) # type: ignore
|
| 337 |
+
# Alternatively, could just do this:
|
| 338 |
+
# trainer.fsdp_model = torch.compile(trainer.fsdp_model, **cfg.compile.asdict())
|
| 339 |
+
|
| 340 |
+
if not cfg.dry_run:
|
| 341 |
+
log.info("Starting training...")
|
| 342 |
+
trainer.fit()
|
| 343 |
+
log.info("Training complete")
|
| 344 |
+
else:
|
| 345 |
+
log.info("Dry run complete")
|
| 346 |
+
|
| 347 |
+
|
| 348 |
+
if __name__ == "__main__":
|
| 349 |
+
try:
|
| 350 |
+
mp.set_start_method("spawn", force=True)
|
| 351 |
+
except RuntimeError as e:
|
| 352 |
+
print(f"failed to set multiprocessing start method: {e}")
|
| 353 |
+
log.info(f"Multiprocessing start method set to '{mp.get_start_method()}'")
|
| 354 |
+
|
| 355 |
+
# Set CUDA device.
|
| 356 |
+
torch.cuda.set_device(f"cuda:{get_local_rank()}")
|
| 357 |
+
|
| 358 |
+
# Initialize process group.
|
| 359 |
+
dist.init_process_group(backend="nccl", timeout=timedelta(minutes=30))
|
| 360 |
+
log.info("Process group initialized")
|
| 361 |
+
|
| 362 |
+
prepare_cli_environment()
|
| 363 |
+
log.info("CLI environment prepared")
|
| 364 |
+
|
| 365 |
+
add_cached_path_clients()
|
| 366 |
+
|
| 367 |
+
try:
|
| 368 |
+
yaml_path, args_list = sys.argv[1], sys.argv[2:]
|
| 369 |
+
except IndexError:
|
| 370 |
+
raise OLMoCliError(f"Usage: {sys.argv[0]} [CONFIG_PATH] [OPTIONS]")
|
| 371 |
+
|
| 372 |
+
cfg = TrainConfig.load(yaml_path, [clean_opt(s) for s in args_list])
|
| 373 |
+
main(cfg)
|
wandb/wandb/run-20251219_032809-fhag09gx/files/config.yaml
ADDED
|
@@ -0,0 +1,883 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
_wandb:
|
| 2 |
+
value:
|
| 3 |
+
cli_version: 0.23.1
|
| 4 |
+
code_path: code/OLMo/scripts/train.py
|
| 5 |
+
e:
|
| 6 |
+
hkx9xgphpmzezm5xyyncldepfa9kpirs:
|
| 7 |
+
args:
|
| 8 |
+
- configs/test/OLMoE-7B-500GB-test.yaml
|
| 9 |
+
- --run_name=olmoe-7b-29b-2ep
|
| 10 |
+
- --save_overwrite
|
| 11 |
+
codePath: OLMo/scripts/train.py
|
| 12 |
+
codePathLocal: scripts/train.py
|
| 13 |
+
cpu_count: 96
|
| 14 |
+
cpu_count_logical: 192
|
| 15 |
+
cudaVersion: "13.0"
|
| 16 |
+
disk:
|
| 17 |
+
/:
|
| 18 |
+
total: "422146228224"
|
| 19 |
+
used: "65563865088"
|
| 20 |
+
email: alexmyasnikov451@gmail.com
|
| 21 |
+
executable: /data/users/xandi281/work/Hi-OLMoE/venv/bin/python3
|
| 22 |
+
git:
|
| 23 |
+
commit: 9b3ad56c62f3b7b0e435efc757c49704db6e6333
|
| 24 |
+
remote: git@github.com:AverageMetaheuristicsEnjoyer/Hi-OLMoE.git
|
| 25 |
+
gpu: NVIDIA H200
|
| 26 |
+
gpu_count: 8
|
| 27 |
+
gpu_nvidia:
|
| 28 |
+
- architecture: Hopper
|
| 29 |
+
cudaCores: 16896
|
| 30 |
+
memoryTotal: "150754820096"
|
| 31 |
+
name: NVIDIA H200
|
| 32 |
+
uuid: GPU-28548a6f-a458-cff0-d4d9-cd8992f7bfc8
|
| 33 |
+
- architecture: Hopper
|
| 34 |
+
cudaCores: 16896
|
| 35 |
+
memoryTotal: "150754820096"
|
| 36 |
+
name: NVIDIA H200
|
| 37 |
+
uuid: GPU-390840f6-de3c-1ecc-1353-cf4d01ab89e2
|
| 38 |
+
- architecture: Hopper
|
| 39 |
+
cudaCores: 16896
|
| 40 |
+
memoryTotal: "150754820096"
|
| 41 |
+
name: NVIDIA H200
|
| 42 |
+
uuid: GPU-bbba09e7-ef87-452c-7d34-cb2a03510f2c
|
| 43 |
+
- architecture: Hopper
|
| 44 |
+
cudaCores: 16896
|
| 45 |
+
memoryTotal: "150754820096"
|
| 46 |
+
name: NVIDIA H200
|
| 47 |
+
uuid: GPU-722903bc-18d9-e3ef-0146-caff9c35e5bf
|
| 48 |
+
- architecture: Hopper
|
| 49 |
+
cudaCores: 16896
|
| 50 |
+
memoryTotal: "150754820096"
|
| 51 |
+
name: NVIDIA H200
|
| 52 |
+
uuid: GPU-f09af96c-4a99-e9b3-2e01-4d03d1a959dd
|
| 53 |
+
- architecture: Hopper
|
| 54 |
+
cudaCores: 16896
|
| 55 |
+
memoryTotal: "150754820096"
|
| 56 |
+
name: NVIDIA H200
|
| 57 |
+
uuid: GPU-0aa55090-5ce5-56e2-62cc-36bc2ae4fb77
|
| 58 |
+
- architecture: Hopper
|
| 59 |
+
cudaCores: 16896
|
| 60 |
+
memoryTotal: "150754820096"
|
| 61 |
+
name: NVIDIA H200
|
| 62 |
+
uuid: GPU-8a81a780-cddb-5b47-490f-871873b601ec
|
| 63 |
+
- architecture: Hopper
|
| 64 |
+
cudaCores: 16896
|
| 65 |
+
memoryTotal: "150754820096"
|
| 66 |
+
name: NVIDIA H200
|
| 67 |
+
uuid: GPU-81dfcc32-ea97-c0f1-778e-ff44c4992c1a
|
| 68 |
+
host: ubuntu-gpu-pod-final
|
| 69 |
+
memory:
|
| 70 |
+
total: "2163971334144"
|
| 71 |
+
os: Linux-5.15.0-160-generic-x86_64-with-glibc2.39
|
| 72 |
+
program: /data/users/xandi281/work/Hi-OLMoE/OLMo/scripts/train.py
|
| 73 |
+
python: CPython 3.11.14
|
| 74 |
+
root: runs/olmoe-7b-29b-2ep/wandb
|
| 75 |
+
startedAt: "2025-12-19T03:28:09.455870Z"
|
| 76 |
+
writerId: hkx9xgphpmzezm5xyyncldepfa9kpirs
|
| 77 |
+
m: []
|
| 78 |
+
python_version: 3.11.14
|
| 79 |
+
t:
|
| 80 |
+
"1":
|
| 81 |
+
- 1
|
| 82 |
+
- 5
|
| 83 |
+
- 11
|
| 84 |
+
- 49
|
| 85 |
+
- 51
|
| 86 |
+
- 53
|
| 87 |
+
"2":
|
| 88 |
+
- 1
|
| 89 |
+
- 5
|
| 90 |
+
- 11
|
| 91 |
+
- 49
|
| 92 |
+
- 51
|
| 93 |
+
- 53
|
| 94 |
+
"3":
|
| 95 |
+
- 2
|
| 96 |
+
- 13
|
| 97 |
+
- 15
|
| 98 |
+
- 16
|
| 99 |
+
- 61
|
| 100 |
+
"4": 3.11.14
|
| 101 |
+
"5": 0.23.1
|
| 102 |
+
"6": 4.57.3
|
| 103 |
+
"10":
|
| 104 |
+
- 19
|
| 105 |
+
"12": 0.23.1
|
| 106 |
+
"13": linux-x86_64
|
| 107 |
+
activation_checkpointing:
|
| 108 |
+
value: fine_grained
|
| 109 |
+
auxiliary_loss_multiplier:
|
| 110 |
+
value: 0.0001
|
| 111 |
+
canceled_check_interval:
|
| 112 |
+
value: 50
|
| 113 |
+
compile:
|
| 114 |
+
value: null
|
| 115 |
+
console_log_interval:
|
| 116 |
+
value: 1
|
| 117 |
+
data:
|
| 118 |
+
value:
|
| 119 |
+
datasets: null
|
| 120 |
+
drop_last: true
|
| 121 |
+
generate_attention_mask: false
|
| 122 |
+
generate_doc_lengths: false
|
| 123 |
+
instance_filter:
|
| 124 |
+
repetition_max_count: 32
|
| 125 |
+
repetition_max_period: 13
|
| 126 |
+
repetition_min_period: 1
|
| 127 |
+
label_mask_paths: null
|
| 128 |
+
memmap_dtype: uint16
|
| 129 |
+
num_workers: 8
|
| 130 |
+
pad_direction: right
|
| 131 |
+
paths:
|
| 132 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy
|
| 133 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy
|
| 134 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy
|
| 135 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy
|
| 136 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy
|
| 137 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy
|
| 138 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy
|
| 139 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy
|
| 140 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy
|
| 141 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy
|
| 142 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy
|
| 143 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy
|
| 144 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy
|
| 145 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-13-00000.npy
|
| 146 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-14-00000.npy
|
| 147 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-15-00000.npy
|
| 148 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-000-00000.npy
|
| 149 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-001-00000.npy
|
| 150 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-002-00000.npy
|
| 151 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-003-00000.npy
|
| 152 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-004-00000.npy
|
| 153 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-005-00000.npy
|
| 154 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-006-00000.npy
|
| 155 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-007-00000.npy
|
| 156 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-008-00000.npy
|
| 157 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-009-00000.npy
|
| 158 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-010-00000.npy
|
| 159 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-011-00000.npy
|
| 160 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-012-00000.npy
|
| 161 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-013-00000.npy
|
| 162 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy
|
| 163 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy
|
| 164 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy
|
| 165 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy
|
| 166 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy
|
| 167 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy
|
| 168 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy
|
| 169 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy
|
| 170 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy
|
| 171 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy
|
| 172 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy
|
| 173 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy
|
| 174 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy
|
| 175 |
+
persistent_workers: true
|
| 176 |
+
pin_memory: true
|
| 177 |
+
prefetch_factor: 8
|
| 178 |
+
seed: null
|
| 179 |
+
timeout: 0
|
| 180 |
+
ddp:
|
| 181 |
+
value: null
|
| 182 |
+
device_eval_batch_size:
|
| 183 |
+
value: 1
|
| 184 |
+
device_train_batch_size:
|
| 185 |
+
value: 256
|
| 186 |
+
device_train_grad_accum:
|
| 187 |
+
value: 128
|
| 188 |
+
device_train_microbatch_size:
|
| 189 |
+
value: 2
|
| 190 |
+
distributed_strategy:
|
| 191 |
+
value: fsdp
|
| 192 |
+
dry_run:
|
| 193 |
+
value: false
|
| 194 |
+
early_stopping_factor:
|
| 195 |
+
value: null
|
| 196 |
+
epoch:
|
| 197 |
+
value: null
|
| 198 |
+
eval_interval:
|
| 199 |
+
value: 2000
|
| 200 |
+
eval_on_load:
|
| 201 |
+
value: false
|
| 202 |
+
eval_subset_num_batches:
|
| 203 |
+
value: -1
|
| 204 |
+
evaluators:
|
| 205 |
+
value:
|
| 206 |
+
- data:
|
| 207 |
+
datasets:
|
| 208 |
+
c4_en-validation:
|
| 209 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/c4_en/val/part-0-00000.npy
|
| 210 |
+
dolma_books-validation:
|
| 211 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_books/val/part-0-00000.npy
|
| 212 |
+
dolma_common-crawl-validation:
|
| 213 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_common-crawl/val/part-0-00000.npy
|
| 214 |
+
dolma_pes2o-validation:
|
| 215 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_pes2o/val/part-0-00000.npy
|
| 216 |
+
dolma_reddit-validation:
|
| 217 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_reddit/val/part-0-00000.npy
|
| 218 |
+
dolma_stack-validation:
|
| 219 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_stack/val/part-0-00000.npy
|
| 220 |
+
dolma_wiki-validation:
|
| 221 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_wiki/val/part-0-00000.npy
|
| 222 |
+
ice-validation:
|
| 223 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/ice/val/part-0-00000.npy
|
| 224 |
+
m2d2_s2orc-validation:
|
| 225 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/m2d2_s2orc/val/part-0-00000.npy
|
| 226 |
+
pile-validation:
|
| 227 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/pile/val/part-0-00000.npy
|
| 228 |
+
wikitext_103-validation:
|
| 229 |
+
- /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/wikitext_103/val/part-0-00000.npy
|
| 230 |
+
drop_last: true
|
| 231 |
+
generate_attention_mask: false
|
| 232 |
+
generate_doc_lengths: false
|
| 233 |
+
instance_filter: null
|
| 234 |
+
label_mask_paths: null
|
| 235 |
+
memmap_dtype: uint16
|
| 236 |
+
num_workers: 0
|
| 237 |
+
pad_direction: right
|
| 238 |
+
paths: null
|
| 239 |
+
persistent_workers: false
|
| 240 |
+
pin_memory: false
|
| 241 |
+
prefetch_factor: null
|
| 242 |
+
seed: null
|
| 243 |
+
timeout: 0
|
| 244 |
+
device_eval_batch_size: null
|
| 245 |
+
label: all-small-ppl-validation
|
| 246 |
+
subset_num_batches: null
|
| 247 |
+
type: lm
|
| 248 |
+
- data:
|
| 249 |
+
datasets: null
|
| 250 |
+
drop_last: false
|
| 251 |
+
generate_attention_mask: false
|
| 252 |
+
generate_doc_lengths: false
|
| 253 |
+
instance_filter: null
|
| 254 |
+
label_mask_paths: null
|
| 255 |
+
memmap_dtype: uint16
|
| 256 |
+
num_workers: 0
|
| 257 |
+
pad_direction: right
|
| 258 |
+
paths: null
|
| 259 |
+
persistent_workers: false
|
| 260 |
+
pin_memory: false
|
| 261 |
+
prefetch_factor: null
|
| 262 |
+
seed: null
|
| 263 |
+
timeout: 0
|
| 264 |
+
device_eval_batch_size: null
|
| 265 |
+
label: piqa
|
| 266 |
+
subset_num_batches: null
|
| 267 |
+
type: downstream
|
| 268 |
+
- data:
|
| 269 |
+
datasets: null
|
| 270 |
+
drop_last: false
|
| 271 |
+
generate_attention_mask: false
|
| 272 |
+
generate_doc_lengths: false
|
| 273 |
+
instance_filter: null
|
| 274 |
+
label_mask_paths: null
|
| 275 |
+
memmap_dtype: uint16
|
| 276 |
+
num_workers: 0
|
| 277 |
+
pad_direction: right
|
| 278 |
+
paths: null
|
| 279 |
+
persistent_workers: false
|
| 280 |
+
pin_memory: false
|
| 281 |
+
prefetch_factor: null
|
| 282 |
+
seed: null
|
| 283 |
+
timeout: 0
|
| 284 |
+
device_eval_batch_size: null
|
| 285 |
+
label: hellaswag
|
| 286 |
+
subset_num_batches: null
|
| 287 |
+
type: downstream
|
| 288 |
+
- data:
|
| 289 |
+
datasets: null
|
| 290 |
+
drop_last: false
|
| 291 |
+
generate_attention_mask: false
|
| 292 |
+
generate_doc_lengths: false
|
| 293 |
+
instance_filter: null
|
| 294 |
+
label_mask_paths: null
|
| 295 |
+
memmap_dtype: uint16
|
| 296 |
+
num_workers: 0
|
| 297 |
+
pad_direction: right
|
| 298 |
+
paths: null
|
| 299 |
+
persistent_workers: false
|
| 300 |
+
pin_memory: false
|
| 301 |
+
prefetch_factor: null
|
| 302 |
+
seed: null
|
| 303 |
+
timeout: 0
|
| 304 |
+
device_eval_batch_size: null
|
| 305 |
+
label: winogrande
|
| 306 |
+
subset_num_batches: null
|
| 307 |
+
type: downstream
|
| 308 |
+
- data:
|
| 309 |
+
datasets: null
|
| 310 |
+
drop_last: false
|
| 311 |
+
generate_attention_mask: false
|
| 312 |
+
generate_doc_lengths: false
|
| 313 |
+
instance_filter: null
|
| 314 |
+
label_mask_paths: null
|
| 315 |
+
memmap_dtype: uint16
|
| 316 |
+
num_workers: 0
|
| 317 |
+
pad_direction: right
|
| 318 |
+
paths: null
|
| 319 |
+
persistent_workers: false
|
| 320 |
+
pin_memory: false
|
| 321 |
+
prefetch_factor: null
|
| 322 |
+
seed: null
|
| 323 |
+
timeout: 0
|
| 324 |
+
device_eval_batch_size: null
|
| 325 |
+
label: openbook_qa
|
| 326 |
+
subset_num_batches: null
|
| 327 |
+
type: downstream
|
| 328 |
+
- data:
|
| 329 |
+
datasets: null
|
| 330 |
+
drop_last: false
|
| 331 |
+
generate_attention_mask: false
|
| 332 |
+
generate_doc_lengths: false
|
| 333 |
+
instance_filter: null
|
| 334 |
+
label_mask_paths: null
|
| 335 |
+
memmap_dtype: uint16
|
| 336 |
+
num_workers: 0
|
| 337 |
+
pad_direction: right
|
| 338 |
+
paths: null
|
| 339 |
+
persistent_workers: false
|
| 340 |
+
pin_memory: false
|
| 341 |
+
prefetch_factor: null
|
| 342 |
+
seed: null
|
| 343 |
+
timeout: 0
|
| 344 |
+
device_eval_batch_size: null
|
| 345 |
+
label: boolq
|
| 346 |
+
subset_num_batches: null
|
| 347 |
+
type: downstream
|
| 348 |
+
- data:
|
| 349 |
+
datasets: null
|
| 350 |
+
drop_last: false
|
| 351 |
+
generate_attention_mask: false
|
| 352 |
+
generate_doc_lengths: false
|
| 353 |
+
instance_filter: null
|
| 354 |
+
label_mask_paths: null
|
| 355 |
+
memmap_dtype: uint16
|
| 356 |
+
num_workers: 0
|
| 357 |
+
pad_direction: right
|
| 358 |
+
paths: null
|
| 359 |
+
persistent_workers: false
|
| 360 |
+
pin_memory: false
|
| 361 |
+
prefetch_factor: null
|
| 362 |
+
seed: null
|
| 363 |
+
timeout: 0
|
| 364 |
+
device_eval_batch_size: null
|
| 365 |
+
label: sciq
|
| 366 |
+
subset_num_batches: null
|
| 367 |
+
type: downstream
|
| 368 |
+
- data:
|
| 369 |
+
datasets: null
|
| 370 |
+
drop_last: false
|
| 371 |
+
generate_attention_mask: false
|
| 372 |
+
generate_doc_lengths: false
|
| 373 |
+
instance_filter: null
|
| 374 |
+
label_mask_paths: null
|
| 375 |
+
memmap_dtype: uint16
|
| 376 |
+
num_workers: 0
|
| 377 |
+
pad_direction: right
|
| 378 |
+
paths: null
|
| 379 |
+
persistent_workers: false
|
| 380 |
+
pin_memory: false
|
| 381 |
+
prefetch_factor: null
|
| 382 |
+
seed: null
|
| 383 |
+
timeout: 0
|
| 384 |
+
device_eval_batch_size: null
|
| 385 |
+
label: arc_easy
|
| 386 |
+
subset_num_batches: null
|
| 387 |
+
type: downstream
|
| 388 |
+
- data:
|
| 389 |
+
datasets: null
|
| 390 |
+
drop_last: false
|
| 391 |
+
generate_attention_mask: false
|
| 392 |
+
generate_doc_lengths: false
|
| 393 |
+
instance_filter: null
|
| 394 |
+
label_mask_paths: null
|
| 395 |
+
memmap_dtype: uint16
|
| 396 |
+
num_workers: 0
|
| 397 |
+
pad_direction: right
|
| 398 |
+
paths: null
|
| 399 |
+
persistent_workers: false
|
| 400 |
+
pin_memory: false
|
| 401 |
+
prefetch_factor: null
|
| 402 |
+
seed: null
|
| 403 |
+
timeout: 0
|
| 404 |
+
device_eval_batch_size: null
|
| 405 |
+
label: arc_challenge
|
| 406 |
+
subset_num_batches: null
|
| 407 |
+
type: downstream
|
| 408 |
+
- data:
|
| 409 |
+
datasets: null
|
| 410 |
+
drop_last: false
|
| 411 |
+
generate_attention_mask: false
|
| 412 |
+
generate_doc_lengths: false
|
| 413 |
+
instance_filter: null
|
| 414 |
+
label_mask_paths: null
|
| 415 |
+
memmap_dtype: uint16
|
| 416 |
+
num_workers: 0
|
| 417 |
+
pad_direction: right
|
| 418 |
+
paths: null
|
| 419 |
+
persistent_workers: false
|
| 420 |
+
pin_memory: false
|
| 421 |
+
prefetch_factor: null
|
| 422 |
+
seed: null
|
| 423 |
+
timeout: 0
|
| 424 |
+
device_eval_batch_size: null
|
| 425 |
+
label: copa
|
| 426 |
+
subset_num_batches: null
|
| 427 |
+
type: downstream
|
| 428 |
+
- data:
|
| 429 |
+
datasets: null
|
| 430 |
+
drop_last: false
|
| 431 |
+
generate_attention_mask: false
|
| 432 |
+
generate_doc_lengths: false
|
| 433 |
+
instance_filter: null
|
| 434 |
+
label_mask_paths: null
|
| 435 |
+
memmap_dtype: uint16
|
| 436 |
+
num_workers: 0
|
| 437 |
+
pad_direction: right
|
| 438 |
+
paths: null
|
| 439 |
+
persistent_workers: false
|
| 440 |
+
pin_memory: false
|
| 441 |
+
prefetch_factor: null
|
| 442 |
+
seed: null
|
| 443 |
+
timeout: 0
|
| 444 |
+
device_eval_batch_size: null
|
| 445 |
+
label: commonsense_qa
|
| 446 |
+
subset_num_batches: null
|
| 447 |
+
type: downstream
|
| 448 |
+
- data:
|
| 449 |
+
datasets: null
|
| 450 |
+
drop_last: false
|
| 451 |
+
generate_attention_mask: false
|
| 452 |
+
generate_doc_lengths: false
|
| 453 |
+
instance_filter: null
|
| 454 |
+
label_mask_paths: null
|
| 455 |
+
memmap_dtype: uint16
|
| 456 |
+
num_workers: 0
|
| 457 |
+
pad_direction: right
|
| 458 |
+
paths: null
|
| 459 |
+
persistent_workers: false
|
| 460 |
+
pin_memory: false
|
| 461 |
+
prefetch_factor: null
|
| 462 |
+
seed: null
|
| 463 |
+
timeout: 0
|
| 464 |
+
device_eval_batch_size: null
|
| 465 |
+
label: social_iqa
|
| 466 |
+
subset_num_batches: null
|
| 467 |
+
type: downstream
|
| 468 |
+
- data:
|
| 469 |
+
datasets: null
|
| 470 |
+
drop_last: false
|
| 471 |
+
generate_attention_mask: false
|
| 472 |
+
generate_doc_lengths: false
|
| 473 |
+
instance_filter: null
|
| 474 |
+
label_mask_paths: null
|
| 475 |
+
memmap_dtype: uint16
|
| 476 |
+
num_workers: 0
|
| 477 |
+
pad_direction: right
|
| 478 |
+
paths: null
|
| 479 |
+
persistent_workers: false
|
| 480 |
+
pin_memory: false
|
| 481 |
+
prefetch_factor: null
|
| 482 |
+
seed: null
|
| 483 |
+
timeout: 0
|
| 484 |
+
device_eval_batch_size: null
|
| 485 |
+
label: mmlu_stem_var
|
| 486 |
+
subset_num_batches: null
|
| 487 |
+
type: downstream
|
| 488 |
+
- data:
|
| 489 |
+
datasets: null
|
| 490 |
+
drop_last: false
|
| 491 |
+
generate_attention_mask: false
|
| 492 |
+
generate_doc_lengths: false
|
| 493 |
+
instance_filter: null
|
| 494 |
+
label_mask_paths: null
|
| 495 |
+
memmap_dtype: uint16
|
| 496 |
+
num_workers: 0
|
| 497 |
+
pad_direction: right
|
| 498 |
+
paths: null
|
| 499 |
+
persistent_workers: false
|
| 500 |
+
pin_memory: false
|
| 501 |
+
prefetch_factor: null
|
| 502 |
+
seed: null
|
| 503 |
+
timeout: 0
|
| 504 |
+
device_eval_batch_size: null
|
| 505 |
+
label: mmlu_humanities_var
|
| 506 |
+
subset_num_batches: null
|
| 507 |
+
type: downstream
|
| 508 |
+
- data:
|
| 509 |
+
datasets: null
|
| 510 |
+
drop_last: false
|
| 511 |
+
generate_attention_mask: false
|
| 512 |
+
generate_doc_lengths: false
|
| 513 |
+
instance_filter: null
|
| 514 |
+
label_mask_paths: null
|
| 515 |
+
memmap_dtype: uint16
|
| 516 |
+
num_workers: 0
|
| 517 |
+
pad_direction: right
|
| 518 |
+
paths: null
|
| 519 |
+
persistent_workers: false
|
| 520 |
+
pin_memory: false
|
| 521 |
+
prefetch_factor: null
|
| 522 |
+
seed: null
|
| 523 |
+
timeout: 0
|
| 524 |
+
device_eval_batch_size: null
|
| 525 |
+
label: mmlu_social_sciences_var
|
| 526 |
+
subset_num_batches: null
|
| 527 |
+
type: downstream
|
| 528 |
+
- data:
|
| 529 |
+
datasets: null
|
| 530 |
+
drop_last: false
|
| 531 |
+
generate_attention_mask: false
|
| 532 |
+
generate_doc_lengths: false
|
| 533 |
+
instance_filter: null
|
| 534 |
+
label_mask_paths: null
|
| 535 |
+
memmap_dtype: uint16
|
| 536 |
+
num_workers: 0
|
| 537 |
+
pad_direction: right
|
| 538 |
+
paths: null
|
| 539 |
+
persistent_workers: false
|
| 540 |
+
pin_memory: false
|
| 541 |
+
prefetch_factor: null
|
| 542 |
+
seed: null
|
| 543 |
+
timeout: 0
|
| 544 |
+
device_eval_batch_size: null
|
| 545 |
+
label: mmlu_other_var
|
| 546 |
+
subset_num_batches: null
|
| 547 |
+
type: downstream
|
| 548 |
+
- data:
|
| 549 |
+
datasets: null
|
| 550 |
+
drop_last: false
|
| 551 |
+
generate_attention_mask: false
|
| 552 |
+
generate_doc_lengths: false
|
| 553 |
+
instance_filter: null
|
| 554 |
+
label_mask_paths: null
|
| 555 |
+
memmap_dtype: uint16
|
| 556 |
+
num_workers: 0
|
| 557 |
+
pad_direction: right
|
| 558 |
+
paths: null
|
| 559 |
+
persistent_workers: false
|
| 560 |
+
pin_memory: false
|
| 561 |
+
prefetch_factor: null
|
| 562 |
+
seed: null
|
| 563 |
+
timeout: 0
|
| 564 |
+
device_eval_batch_size: null
|
| 565 |
+
label: mmlu_stem_mc_5shot
|
| 566 |
+
subset_num_batches: null
|
| 567 |
+
type: downstream
|
| 568 |
+
- data:
|
| 569 |
+
datasets: null
|
| 570 |
+
drop_last: false
|
| 571 |
+
generate_attention_mask: false
|
| 572 |
+
generate_doc_lengths: false
|
| 573 |
+
instance_filter: null
|
| 574 |
+
label_mask_paths: null
|
| 575 |
+
memmap_dtype: uint16
|
| 576 |
+
num_workers: 0
|
| 577 |
+
pad_direction: right
|
| 578 |
+
paths: null
|
| 579 |
+
persistent_workers: false
|
| 580 |
+
pin_memory: false
|
| 581 |
+
prefetch_factor: null
|
| 582 |
+
seed: null
|
| 583 |
+
timeout: 0
|
| 584 |
+
device_eval_batch_size: null
|
| 585 |
+
label: mmlu_humanities_mc_5shot
|
| 586 |
+
subset_num_batches: null
|
| 587 |
+
type: downstream
|
| 588 |
+
- data:
|
| 589 |
+
datasets: null
|
| 590 |
+
drop_last: false
|
| 591 |
+
generate_attention_mask: false
|
| 592 |
+
generate_doc_lengths: false
|
| 593 |
+
instance_filter: null
|
| 594 |
+
label_mask_paths: null
|
| 595 |
+
memmap_dtype: uint16
|
| 596 |
+
num_workers: 0
|
| 597 |
+
pad_direction: right
|
| 598 |
+
paths: null
|
| 599 |
+
persistent_workers: false
|
| 600 |
+
pin_memory: false
|
| 601 |
+
prefetch_factor: null
|
| 602 |
+
seed: null
|
| 603 |
+
timeout: 0
|
| 604 |
+
device_eval_batch_size: null
|
| 605 |
+
label: mmlu_social_sciences_mc_5shot
|
| 606 |
+
subset_num_batches: null
|
| 607 |
+
type: downstream
|
| 608 |
+
- data:
|
| 609 |
+
datasets: null
|
| 610 |
+
drop_last: false
|
| 611 |
+
generate_attention_mask: false
|
| 612 |
+
generate_doc_lengths: false
|
| 613 |
+
instance_filter: null
|
| 614 |
+
label_mask_paths: null
|
| 615 |
+
memmap_dtype: uint16
|
| 616 |
+
num_workers: 0
|
| 617 |
+
pad_direction: right
|
| 618 |
+
paths: null
|
| 619 |
+
persistent_workers: false
|
| 620 |
+
pin_memory: false
|
| 621 |
+
prefetch_factor: null
|
| 622 |
+
seed: null
|
| 623 |
+
timeout: 0
|
| 624 |
+
device_eval_batch_size: null
|
| 625 |
+
label: mmlu_other_mc_5shot
|
| 626 |
+
subset_num_batches: null
|
| 627 |
+
type: downstream
|
| 628 |
+
- data:
|
| 629 |
+
datasets: null
|
| 630 |
+
drop_last: false
|
| 631 |
+
generate_attention_mask: false
|
| 632 |
+
generate_doc_lengths: false
|
| 633 |
+
instance_filter: null
|
| 634 |
+
label_mask_paths: null
|
| 635 |
+
memmap_dtype: uint16
|
| 636 |
+
num_workers: 0
|
| 637 |
+
pad_direction: right
|
| 638 |
+
paths: null
|
| 639 |
+
persistent_workers: false
|
| 640 |
+
pin_memory: false
|
| 641 |
+
prefetch_factor: null
|
| 642 |
+
seed: null
|
| 643 |
+
timeout: 0
|
| 644 |
+
device_eval_batch_size: null
|
| 645 |
+
label: mmlu_stem_mc_5shot_test
|
| 646 |
+
subset_num_batches: null
|
| 647 |
+
type: downstream
|
| 648 |
+
- data:
|
| 649 |
+
datasets: null
|
| 650 |
+
drop_last: false
|
| 651 |
+
generate_attention_mask: false
|
| 652 |
+
generate_doc_lengths: false
|
| 653 |
+
instance_filter: null
|
| 654 |
+
label_mask_paths: null
|
| 655 |
+
memmap_dtype: uint16
|
| 656 |
+
num_workers: 0
|
| 657 |
+
pad_direction: right
|
| 658 |
+
paths: null
|
| 659 |
+
persistent_workers: false
|
| 660 |
+
pin_memory: false
|
| 661 |
+
prefetch_factor: null
|
| 662 |
+
seed: null
|
| 663 |
+
timeout: 0
|
| 664 |
+
device_eval_batch_size: null
|
| 665 |
+
label: mmlu_humanities_mc_5shot_test
|
| 666 |
+
subset_num_batches: null
|
| 667 |
+
type: downstream
|
| 668 |
+
- data:
|
| 669 |
+
datasets: null
|
| 670 |
+
drop_last: false
|
| 671 |
+
generate_attention_mask: false
|
| 672 |
+
generate_doc_lengths: false
|
| 673 |
+
instance_filter: null
|
| 674 |
+
label_mask_paths: null
|
| 675 |
+
memmap_dtype: uint16
|
| 676 |
+
num_workers: 0
|
| 677 |
+
pad_direction: right
|
| 678 |
+
paths: null
|
| 679 |
+
persistent_workers: false
|
| 680 |
+
pin_memory: false
|
| 681 |
+
prefetch_factor: null
|
| 682 |
+
seed: null
|
| 683 |
+
timeout: 0
|
| 684 |
+
device_eval_batch_size: null
|
| 685 |
+
label: mmlu_social_sciences_mc_5shot_test
|
| 686 |
+
subset_num_batches: null
|
| 687 |
+
type: downstream
|
| 688 |
+
- data:
|
| 689 |
+
datasets: null
|
| 690 |
+
drop_last: false
|
| 691 |
+
generate_attention_mask: false
|
| 692 |
+
generate_doc_lengths: false
|
| 693 |
+
instance_filter: null
|
| 694 |
+
label_mask_paths: null
|
| 695 |
+
memmap_dtype: uint16
|
| 696 |
+
num_workers: 0
|
| 697 |
+
pad_direction: right
|
| 698 |
+
paths: null
|
| 699 |
+
persistent_workers: false
|
| 700 |
+
pin_memory: false
|
| 701 |
+
prefetch_factor: null
|
| 702 |
+
seed: null
|
| 703 |
+
timeout: 0
|
| 704 |
+
device_eval_batch_size: null
|
| 705 |
+
label: mmlu_other_mc_5shot_test
|
| 706 |
+
subset_num_batches: null
|
| 707 |
+
type: downstream
|
| 708 |
+
extra_steps_after_cancel:
|
| 709 |
+
value: 10
|
| 710 |
+
fast_forward_batches:
|
| 711 |
+
value: null
|
| 712 |
+
force_save_unsharded:
|
| 713 |
+
value: false
|
| 714 |
+
fsdp:
|
| 715 |
+
value:
|
| 716 |
+
hybrid_sharding_num_model_replicas: null
|
| 717 |
+
precision: mixed
|
| 718 |
+
sharding_strategy: FULL_SHARD
|
| 719 |
+
use_orig_params: true
|
| 720 |
+
wrapping_strategy: by_block
|
| 721 |
+
fused_loss:
|
| 722 |
+
value: null
|
| 723 |
+
gen1_gc_interval:
|
| 724 |
+
value: 1
|
| 725 |
+
global_train_batch_size:
|
| 726 |
+
value: 1024
|
| 727 |
+
hf_datasets_cache_dir:
|
| 728 |
+
value: null
|
| 729 |
+
load_path:
|
| 730 |
+
value: null
|
| 731 |
+
load_path_sharded_checkpointer:
|
| 732 |
+
value: null
|
| 733 |
+
max_duration:
|
| 734 |
+
value: 2ep
|
| 735 |
+
max_grad_norm:
|
| 736 |
+
value: 1
|
| 737 |
+
max_grad_norm_ratio:
|
| 738 |
+
value: null
|
| 739 |
+
model:
|
| 740 |
+
value:
|
| 741 |
+
activation_type: swiglu
|
| 742 |
+
alibi: false
|
| 743 |
+
alibi_bias_max: 8
|
| 744 |
+
attention_dropout: 0
|
| 745 |
+
attention_layer_norm: true
|
| 746 |
+
attention_layer_norm_with_affine: true
|
| 747 |
+
bias_for_layer_norm: false
|
| 748 |
+
block_group_size: 1
|
| 749 |
+
block_type: moe
|
| 750 |
+
clip_qkv: null
|
| 751 |
+
d_model: 2048
|
| 752 |
+
emb_init_std: null
|
| 753 |
+
embedding_dropout: 0
|
| 754 |
+
embedding_layer_norm: false
|
| 755 |
+
embedding_size: 50304
|
| 756 |
+
eos_token_id: 0
|
| 757 |
+
flash_attention: true
|
| 758 |
+
himoe_enabled: false
|
| 759 |
+
himoe_lambda1: 0.01
|
| 760 |
+
himoe_lambda2: 0.1
|
| 761 |
+
himoe_n_groups: 4
|
| 762 |
+
himoe_top_k_per_group: 2
|
| 763 |
+
include_bias: false
|
| 764 |
+
init_cutoff_factor: 3
|
| 765 |
+
init_device: meta
|
| 766 |
+
init_fn: normal
|
| 767 |
+
init_std: 0.02
|
| 768 |
+
layer_norm_eps: 1e-05
|
| 769 |
+
layer_norm_type: rms
|
| 770 |
+
layer_norm_with_affine: true
|
| 771 |
+
max_sequence_length: 4096
|
| 772 |
+
mlp_hidden_size: null
|
| 773 |
+
mlp_ratio: 1
|
| 774 |
+
moe_capacity_factor: 1.25
|
| 775 |
+
moe_dropless: false
|
| 776 |
+
moe_lbl_in_fp32: false
|
| 777 |
+
moe_log_expert_assignment: false
|
| 778 |
+
moe_loss_weight: 0.01
|
| 779 |
+
moe_mlp_impl: sparse
|
| 780 |
+
moe_num_experts: 64
|
| 781 |
+
moe_shared_expert: false
|
| 782 |
+
moe_top_k: 8
|
| 783 |
+
moe_zloss_weight: 0.001
|
| 784 |
+
multi_query_attention: null
|
| 785 |
+
n_heads: 16
|
| 786 |
+
n_kv_heads: null
|
| 787 |
+
n_layers: 16
|
| 788 |
+
norm_after: false
|
| 789 |
+
pad_token_id: 1
|
| 790 |
+
precision: amp_bf16
|
| 791 |
+
residual_dropout: 0
|
| 792 |
+
rope: true
|
| 793 |
+
rope_full_precision: true
|
| 794 |
+
rope_theta: 10000
|
| 795 |
+
scale_emb_init: false
|
| 796 |
+
scale_logits: false
|
| 797 |
+
vocab_size: 50280
|
| 798 |
+
weight_tying: false
|
| 799 |
+
module_outputs_save_steps:
|
| 800 |
+
value: null
|
| 801 |
+
new_style_checkpoints:
|
| 802 |
+
value: null
|
| 803 |
+
no_pre_train_checkpoint:
|
| 804 |
+
value: true
|
| 805 |
+
optimizer:
|
| 806 |
+
value:
|
| 807 |
+
betas:
|
| 808 |
+
- 0.9
|
| 809 |
+
- 0.95
|
| 810 |
+
decay_embeddings: true
|
| 811 |
+
decay_norm_and_bias: true
|
| 812 |
+
eps: 1e-08
|
| 813 |
+
learning_rate: 0.0004
|
| 814 |
+
metrics_log_interval: 10
|
| 815 |
+
name: adamw
|
| 816 |
+
no_decay_norm_and_bias: null
|
| 817 |
+
record_update_metrics: false
|
| 818 |
+
selective_updates: false
|
| 819 |
+
weight_decay: 0.1
|
| 820 |
+
precision:
|
| 821 |
+
value: amp_bf16
|
| 822 |
+
python_profiling:
|
| 823 |
+
value: false
|
| 824 |
+
remote_save_folder:
|
| 825 |
+
value: null
|
| 826 |
+
reset_optimizer_state:
|
| 827 |
+
value: false
|
| 828 |
+
reset_trainer_state:
|
| 829 |
+
value: false
|
| 830 |
+
restore_dataloader:
|
| 831 |
+
value: true
|
| 832 |
+
run_name:
|
| 833 |
+
value: olmoe-7b-29b-2ep
|
| 834 |
+
save_data_indices:
|
| 835 |
+
value: true
|
| 836 |
+
save_folder:
|
| 837 |
+
value: runs/olmoe-7b-29b-2ep
|
| 838 |
+
save_interval:
|
| 839 |
+
value: 2000
|
| 840 |
+
save_interval_ephemeral:
|
| 841 |
+
value: null
|
| 842 |
+
save_interval_unsharded:
|
| 843 |
+
value: null
|
| 844 |
+
save_num_checkpoints_to_keep:
|
| 845 |
+
value: 5
|
| 846 |
+
save_num_unsharded_checkpoints_to_keep:
|
| 847 |
+
value: -1
|
| 848 |
+
save_overwrite:
|
| 849 |
+
value: true
|
| 850 |
+
scheduler:
|
| 851 |
+
value:
|
| 852 |
+
alpha_f: 0.1
|
| 853 |
+
grad_clip_warmup_factor: null
|
| 854 |
+
grad_clip_warmup_steps: null
|
| 855 |
+
name: cosine_with_warmup
|
| 856 |
+
t_max: 58540000000
|
| 857 |
+
t_warmup: 1405296000
|
| 858 |
+
units: tokens
|
| 859 |
+
warmup_min_lr: null
|
| 860 |
+
seed:
|
| 861 |
+
value: 6198
|
| 862 |
+
sharded_checkpointer:
|
| 863 |
+
value: olmo_core
|
| 864 |
+
softmax_auxiliary_loss:
|
| 865 |
+
value: false
|
| 866 |
+
speed_monitor:
|
| 867 |
+
value:
|
| 868 |
+
gpu_flops_available: null
|
| 869 |
+
window_size: 1
|
| 870 |
+
stop_after:
|
| 871 |
+
value: null
|
| 872 |
+
stop_at:
|
| 873 |
+
value: null
|
| 874 |
+
time_limit:
|
| 875 |
+
value: null
|
| 876 |
+
tokenizer:
|
| 877 |
+
value:
|
| 878 |
+
identifier: tokenizers/allenai_gpt-neox-olmo-dolma-v1_5.json
|
| 879 |
+
truncate_direction: right
|
| 880 |
+
torch_profiling:
|
| 881 |
+
value: false
|
| 882 |
+
try_load_latest_save:
|
| 883 |
+
value: false
|
wandb/wandb/run-20251219_032809-fhag09gx/files/output.log
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f0d408b475b86f6651a715c0f8d1688b1bcd94ecb011cfffa66ed89399020aa5
|
| 3 |
+
size 36174013
|
wandb/wandb/run-20251219_032809-fhag09gx/files/requirements.txt
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
twine==6.2.0
|
| 2 |
+
Pygments==2.19.2
|
| 3 |
+
rsa==4.9.1
|
| 4 |
+
black==23.12.1
|
| 5 |
+
pyasn1_modules==0.4.2
|
| 6 |
+
six==1.17.0
|
| 7 |
+
nvidia-cufft==12.0.0.15
|
| 8 |
+
pytest==9.0.2
|
| 9 |
+
annotated-types==0.7.0
|
| 10 |
+
id==1.5.0
|
| 11 |
+
MarkupSafe==2.1.5
|
| 12 |
+
wandb==0.23.1
|
| 13 |
+
click-option-group==0.5.9
|
| 14 |
+
petname==2.6
|
| 15 |
+
ai2-olmo-core==0.1.0
|
| 16 |
+
wheel==0.45.1
|
| 17 |
+
pyarrow==22.0.0
|
| 18 |
+
dataclass-extensions==0.2.4
|
| 19 |
+
gitdb==4.0.12
|
| 20 |
+
ftfy==6.3.1
|
| 21 |
+
nvidia-nvshmem-cu13==3.3.24
|
| 22 |
+
necessary==0.4.3
|
| 23 |
+
ai2-olmo==0.5.0
|
| 24 |
+
httpx==0.28.1
|
| 25 |
+
lightning-utilities==0.15.2
|
| 26 |
+
pytz==2025.2
|
| 27 |
+
frozenlist==1.8.0
|
| 28 |
+
hf-xet==1.2.0
|
| 29 |
+
torchmetrics==1.8.2
|
| 30 |
+
pytest-sphinx==0.6.3
|
| 31 |
+
pydantic==2.12.5
|
| 32 |
+
platformdirs==4.5.1
|
| 33 |
+
urllib3==2.6.2
|
| 34 |
+
iniconfig==2.3.0
|
| 35 |
+
h11==0.16.0
|
| 36 |
+
googleapis-common-protos==1.72.0
|
| 37 |
+
nvidia-nvtx==13.0.39
|
| 38 |
+
beaker-gantry==3.3.0
|
| 39 |
+
nvidia-nvjitlink==13.0.39
|
| 40 |
+
xxhash==3.6.0
|
| 41 |
+
grpcio==1.76.0
|
| 42 |
+
glom==24.11.0
|
| 43 |
+
pycparser==2.23
|
| 44 |
+
msgspec==0.20.0
|
| 45 |
+
huggingface-hub==0.36.0
|
| 46 |
+
protobuf==6.33.2
|
| 47 |
+
mdurl==0.1.2
|
| 48 |
+
triton==3.5.1
|
| 49 |
+
python-dateutil==2.9.0.post0
|
| 50 |
+
megablocks==0.5.1
|
| 51 |
+
propcache==0.4.1
|
| 52 |
+
nvidia-cuda-runtime==13.0.48
|
| 53 |
+
readme_renderer==44.0
|
| 54 |
+
fsspec==2025.10.0
|
| 55 |
+
PyYAML==6.0.3
|
| 56 |
+
face==24.0.0
|
| 57 |
+
numpy==1.26.4
|
| 58 |
+
more-itertools==10.8.0
|
| 59 |
+
nvidia-cuda-nvrtc==13.0.48
|
| 60 |
+
docutils==0.22.3
|
| 61 |
+
tokenizers==0.22.1
|
| 62 |
+
smmap==5.0.2
|
| 63 |
+
jaraco.classes==3.4.0
|
| 64 |
+
google-resumable-media==2.8.0
|
| 65 |
+
trouting==0.3.3
|
| 66 |
+
s3transfer==0.16.0
|
| 67 |
+
keyring==25.7.0
|
| 68 |
+
typing-inspection==0.4.2
|
| 69 |
+
requirements-parser==0.13.0
|
| 70 |
+
isort==5.12.0
|
| 71 |
+
wrapt==2.0.1
|
| 72 |
+
google-api-core==2.28.1
|
| 73 |
+
beaker-py==2.5.3
|
| 74 |
+
jeepney==0.9.0
|
| 75 |
+
rfc3986==2.0.0
|
| 76 |
+
tzdata==2025.2
|
| 77 |
+
cachetools==6.2.2
|
| 78 |
+
torch==2.9.1+cu130
|
| 79 |
+
networkx==3.6.1
|
| 80 |
+
google-crc32c==1.7.1
|
| 81 |
+
click==8.3.1
|
| 82 |
+
google-auth==2.43.0
|
| 83 |
+
smart_open==7.5.0
|
| 84 |
+
pydantic_core==2.41.5
|
| 85 |
+
tomli==2.3.0
|
| 86 |
+
markdown-it-py==4.0.0
|
| 87 |
+
sentry-sdk==2.47.0
|
| 88 |
+
tqdm==4.67.1
|
| 89 |
+
wcwidth==0.2.14
|
| 90 |
+
importlib_resources==6.5.2
|
| 91 |
+
stanford-stk==0.7.0
|
| 92 |
+
boltons==25.0.0
|
| 93 |
+
GitPython==3.1.45
|
| 94 |
+
nvidia-curand==10.4.0.35
|
| 95 |
+
cryptography==46.0.3
|
| 96 |
+
nvidia-cuda-cupti==13.0.48
|
| 97 |
+
zipp==3.23.0
|
| 98 |
+
boto3==1.42.8
|
| 99 |
+
aiohappyeyeballs==2.6.1
|
| 100 |
+
build==1.3.0
|
| 101 |
+
proto-plus==1.26.1
|
| 102 |
+
ruff==0.14.9
|
| 103 |
+
filelock==3.20.0
|
| 104 |
+
Jinja2==3.1.6
|
| 105 |
+
pandas==2.3.3
|
| 106 |
+
backports.tarfile==1.2.0
|
| 107 |
+
google-cloud-core==2.5.0
|
| 108 |
+
pyasn1==0.6.1
|
| 109 |
+
nvidia-cusparse==12.6.2.49
|
| 110 |
+
charset-normalizer==3.4.4
|
| 111 |
+
transformers==4.57.3
|
| 112 |
+
jaraco.context==6.0.1
|
| 113 |
+
joblib==1.5.2
|
| 114 |
+
regex==2025.11.3
|
| 115 |
+
nvidia-nccl-cu13==2.27.7
|
| 116 |
+
multidict==6.7.0
|
| 117 |
+
jaraco.functools==4.3.0
|
| 118 |
+
omegaconf==2.3.0
|
| 119 |
+
mypy_extensions==1.1.0
|
| 120 |
+
attrs==25.4.0
|
| 121 |
+
nvidia-cublas==13.0.0.19
|
| 122 |
+
nvidia-cufile==1.15.0.42
|
| 123 |
+
pathspec==0.12.1
|
| 124 |
+
botocore==1.42.8
|
| 125 |
+
click-help-colors==0.9.4
|
| 126 |
+
pluggy==1.6.0
|
| 127 |
+
multiprocess==0.70.18
|
| 128 |
+
idna==3.11
|
| 129 |
+
aiosignal==1.4.0
|
| 130 |
+
nh3==0.3.2
|
| 131 |
+
nvidia-cusolver==12.0.3.29
|
| 132 |
+
certifi==2025.11.12
|
| 133 |
+
antlr4-python3-runtime==4.9.3
|
| 134 |
+
httpcore==1.0.9
|
| 135 |
+
google-cloud-storage==2.19.0
|
| 136 |
+
threadpoolctl==3.6.0
|
| 137 |
+
requests==2.32.5
|
| 138 |
+
pyproject_hooks==1.2.0
|
| 139 |
+
scikit-learn==1.8.0
|
| 140 |
+
smashed==0.21.5
|
| 141 |
+
requests-toolbelt==1.0.0
|
| 142 |
+
importlib_metadata==8.7.0
|
| 143 |
+
anyio==4.12.0
|
| 144 |
+
cached_path==1.8.0
|
| 145 |
+
mypy==1.3.0
|
| 146 |
+
jmespath==1.0.1
|
| 147 |
+
pip==24.0
|
| 148 |
+
SecretStorage==3.5.0
|
| 149 |
+
nvidia-cudnn-cu13==9.13.0.50
|
| 150 |
+
aiohttp==3.13.2
|
| 151 |
+
typing_extensions==4.15.0
|
| 152 |
+
yarl==1.22.0
|
| 153 |
+
mpmath==1.3.0
|
| 154 |
+
packaging==24.1
|
| 155 |
+
safetensors==0.7.0
|
| 156 |
+
sympy==1.14.0
|
| 157 |
+
scipy==1.16.3
|
| 158 |
+
datasets==4.4.1
|
| 159 |
+
dill==0.4.0
|
| 160 |
+
nvidia-cusparselt-cu13==0.8.0
|
| 161 |
+
setuptools==79.0.1
|
| 162 |
+
rich==13.9.4
|
| 163 |
+
cffi==2.0.0
|
wandb/wandb/run-20251219_032809-fhag09gx/files/wandb-metadata.json
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"os": "Linux-5.15.0-160-generic-x86_64-with-glibc2.39",
|
| 3 |
+
"python": "CPython 3.11.14",
|
| 4 |
+
"startedAt": "2025-12-19T03:28:09.455870Z",
|
| 5 |
+
"args": [
|
| 6 |
+
"configs/test/OLMoE-7B-500GB-test.yaml",
|
| 7 |
+
"--run_name=olmoe-7b-29b-2ep",
|
| 8 |
+
"--save_overwrite"
|
| 9 |
+
],
|
| 10 |
+
"program": "/data/users/xandi281/work/Hi-OLMoE/OLMo/scripts/train.py",
|
| 11 |
+
"codePath": "OLMo/scripts/train.py",
|
| 12 |
+
"codePathLocal": "scripts/train.py",
|
| 13 |
+
"git": {
|
| 14 |
+
"remote": "git@github.com:AverageMetaheuristicsEnjoyer/Hi-OLMoE.git",
|
| 15 |
+
"commit": "9b3ad56c62f3b7b0e435efc757c49704db6e6333"
|
| 16 |
+
},
|
| 17 |
+
"email": "alexmyasnikov451@gmail.com",
|
| 18 |
+
"root": "runs/olmoe-7b-29b-2ep/wandb",
|
| 19 |
+
"host": "ubuntu-gpu-pod-final",
|
| 20 |
+
"executable": "/data/users/xandi281/work/Hi-OLMoE/venv/bin/python3",
|
| 21 |
+
"cpu_count": 96,
|
| 22 |
+
"cpu_count_logical": 192,
|
| 23 |
+
"gpu": "NVIDIA H200",
|
| 24 |
+
"gpu_count": 8,
|
| 25 |
+
"disk": {
|
| 26 |
+
"/": {
|
| 27 |
+
"total": "422146228224",
|
| 28 |
+
"used": "65563865088"
|
| 29 |
+
}
|
| 30 |
+
},
|
| 31 |
+
"memory": {
|
| 32 |
+
"total": "2163971334144"
|
| 33 |
+
},
|
| 34 |
+
"gpu_nvidia": [
|
| 35 |
+
{
|
| 36 |
+
"name": "NVIDIA H200",
|
| 37 |
+
"memoryTotal": "150754820096",
|
| 38 |
+
"cudaCores": 16896,
|
| 39 |
+
"architecture": "Hopper",
|
| 40 |
+
"uuid": "GPU-28548a6f-a458-cff0-d4d9-cd8992f7bfc8"
|
| 41 |
+
},
|
| 42 |
+
{
|
| 43 |
+
"name": "NVIDIA H200",
|
| 44 |
+
"memoryTotal": "150754820096",
|
| 45 |
+
"cudaCores": 16896,
|
| 46 |
+
"architecture": "Hopper",
|
| 47 |
+
"uuid": "GPU-390840f6-de3c-1ecc-1353-cf4d01ab89e2"
|
| 48 |
+
},
|
| 49 |
+
{
|
| 50 |
+
"name": "NVIDIA H200",
|
| 51 |
+
"memoryTotal": "150754820096",
|
| 52 |
+
"cudaCores": 16896,
|
| 53 |
+
"architecture": "Hopper",
|
| 54 |
+
"uuid": "GPU-bbba09e7-ef87-452c-7d34-cb2a03510f2c"
|
| 55 |
+
},
|
| 56 |
+
{
|
| 57 |
+
"name": "NVIDIA H200",
|
| 58 |
+
"memoryTotal": "150754820096",
|
| 59 |
+
"cudaCores": 16896,
|
| 60 |
+
"architecture": "Hopper",
|
| 61 |
+
"uuid": "GPU-722903bc-18d9-e3ef-0146-caff9c35e5bf"
|
| 62 |
+
},
|
| 63 |
+
{
|
| 64 |
+
"name": "NVIDIA H200",
|
| 65 |
+
"memoryTotal": "150754820096",
|
| 66 |
+
"cudaCores": 16896,
|
| 67 |
+
"architecture": "Hopper",
|
| 68 |
+
"uuid": "GPU-f09af96c-4a99-e9b3-2e01-4d03d1a959dd"
|
| 69 |
+
},
|
| 70 |
+
{
|
| 71 |
+
"name": "NVIDIA H200",
|
| 72 |
+
"memoryTotal": "150754820096",
|
| 73 |
+
"cudaCores": 16896,
|
| 74 |
+
"architecture": "Hopper",
|
| 75 |
+
"uuid": "GPU-0aa55090-5ce5-56e2-62cc-36bc2ae4fb77"
|
| 76 |
+
},
|
| 77 |
+
{
|
| 78 |
+
"name": "NVIDIA H200",
|
| 79 |
+
"memoryTotal": "150754820096",
|
| 80 |
+
"cudaCores": 16896,
|
| 81 |
+
"architecture": "Hopper",
|
| 82 |
+
"uuid": "GPU-8a81a780-cddb-5b47-490f-871873b601ec"
|
| 83 |
+
},
|
| 84 |
+
{
|
| 85 |
+
"name": "NVIDIA H200",
|
| 86 |
+
"memoryTotal": "150754820096",
|
| 87 |
+
"cudaCores": 16896,
|
| 88 |
+
"architecture": "Hopper",
|
| 89 |
+
"uuid": "GPU-81dfcc32-ea97-c0f1-778e-ff44c4992c1a"
|
| 90 |
+
}
|
| 91 |
+
],
|
| 92 |
+
"cudaVersion": "13.0",
|
| 93 |
+
"writerId": "hkx9xgphpmzezm5xyyncldepfa9kpirs"
|
| 94 |
+
}
|
wandb/wandb/run-20251219_032809-fhag09gx/files/wandb-summary.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
wandb/wandb/run-20251219_032809-fhag09gx/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2025-12-19T03:28:09.715068381Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
|
| 2 |
+
{"time":"2025-12-19T03:28:10.010671632Z","level":"INFO","msg":"stream: created new stream","id":"fhag09gx"}
|
| 3 |
+
{"time":"2025-12-19T03:28:10.01079177Z","level":"INFO","msg":"writer: started","stream_id":"fhag09gx"}
|
| 4 |
+
{"time":"2025-12-19T03:28:10.010792247Z","level":"INFO","msg":"sender: started","stream_id":"fhag09gx"}
|
| 5 |
+
{"time":"2025-12-19T03:28:10.011088687Z","level":"INFO","msg":"handler: started","stream_id":"fhag09gx"}
|
| 6 |
+
{"time":"2025-12-19T03:28:10.011242775Z","level":"INFO","msg":"stream: started","id":"fhag09gx"}
|
| 7 |
+
{"time":"2025-12-19T03:28:10.069884068Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
|
| 8 |
+
{"time":"2025-12-25T13:58:15.121256706Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
|
| 9 |
+
{"time":"2025-12-25T13:58:15.388114558Z","level":"INFO","msg":"handler: operation stats","stats":{}}
|
| 10 |
+
{"time":"2025-12-25T13:58:15.404292716Z","level":"INFO","msg":"stream: closing","id":"fhag09gx"}
|
| 11 |
+
{"time":"2025-12-25T13:58:15.404311317Z","level":"INFO","msg":"handler: closed","stream_id":"fhag09gx"}
|
| 12 |
+
{"time":"2025-12-25T13:58:15.404690567Z","level":"INFO","msg":"sender: closed","stream_id":"fhag09gx"}
|
| 13 |
+
{"time":"2025-12-25T13:58:15.404702005Z","level":"INFO","msg":"stream: closed","id":"fhag09gx"}
|