diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..c8ab63ac9f8d5bf09e0597e15cbd1b71f55014b7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +wandb/wandb/run-20251219_032809-fhag09gx/files/output.log filter=lfs diff=lfs merge=lfs -text diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6b22cf92b1fe7b12877ad660e421dcc0c65ffa16 --- /dev/null +++ b/config.yaml @@ -0,0 +1,724 @@ +run_name: olmoe-7b-29b-2ep +seed: 6198 +epoch: null +dry_run: false +model: + d_model: 2048 + n_heads: 16 + n_kv_heads: null + clip_qkv: null + n_layers: 16 + mlp_ratio: 1 + mlp_hidden_size: null + activation_type: swiglu + block_type: moe + block_group_size: 1 + alibi: false + alibi_bias_max: 8.0 + rope: true + rope_full_precision: true + rope_theta: 10000 + flash_attention: true + attention_dropout: 0.0 + multi_query_attention: null + attention_layer_norm: true + residual_dropout: 0.0 + embedding_dropout: 0.0 + embedding_layer_norm: false + layer_norm_type: rms + layer_norm_with_affine: true + layer_norm_eps: 1.0e-05 + attention_layer_norm_with_affine: true + max_sequence_length: 4096 + include_bias: false + bias_for_layer_norm: false + scale_logits: false + vocab_size: 50280 + embedding_size: 50304 + weight_tying: false + eos_token_id: 0 + pad_token_id: 1 + init_device: meta + init_fn: normal + init_std: 0.02 + init_cutoff_factor: 3.0 + precision: amp_bf16 + moe_num_experts: 64 + moe_top_k: 8 + moe_mlp_impl: sparse + moe_log_expert_assignment: false + moe_shared_expert: false + moe_lbl_in_fp32: false + moe_loss_weight: 0.01 + moe_zloss_weight: 0.001 + moe_dropless: false + moe_capacity_factor: 1.25 + himoe_enabled: false + himoe_n_groups: 4 + himoe_top_k_per_group: 2 + himoe_lambda1: 0.01 + himoe_lambda2: 0.1 + scale_emb_init: false + emb_init_std: null + norm_after: false +optimizer: + name: adamw + learning_rate: 0.0004 + weight_decay: 0.1 + betas: + - 0.9 + - 0.95 + eps: 1.0e-08 + no_decay_norm_and_bias: null + selective_updates: false + decay_norm_and_bias: true + decay_embeddings: true + metrics_log_interval: 10 + record_update_metrics: false +scheduler: + name: cosine_with_warmup + units: tokens + t_warmup: 1405296000 + t_max: 58540000000 + alpha_f: 0.1 + grad_clip_warmup_steps: null + grad_clip_warmup_factor: null + warmup_min_lr: null +data: + paths: + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-13-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-14-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-15-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-000-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-001-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-002-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-003-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-004-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-005-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-006-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-007-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-008-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-009-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-010-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-011-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-012-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-013-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 8 + drop_last: true + pin_memory: true + prefetch_factor: 8 + persistent_workers: true + timeout: 0 + seed: null + instance_filter: + repetition_max_period: 13 + repetition_min_period: 1 + repetition_max_count: 32 +restore_dataloader: true +fast_forward_batches: null +evaluators: +- label: all-small-ppl-validation + type: lm + data: + paths: null + memmap_dtype: uint16 + datasets: + c4_en-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/c4_en/val/part-0-00000.npy + dolma_books-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_books/val/part-0-00000.npy + dolma_common-crawl-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_common-crawl/val/part-0-00000.npy + dolma_pes2o-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_pes2o/val/part-0-00000.npy + dolma_reddit-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_reddit/val/part-0-00000.npy + dolma_stack-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_stack/val/part-0-00000.npy + dolma_wiki-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_wiki/val/part-0-00000.npy + ice-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/ice/val/part-0-00000.npy + m2d2_s2orc-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/m2d2_s2orc/val/part-0-00000.npy + pile-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/pile/val/part-0-00000.npy + wikitext_103-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/wikitext_103/val/part-0-00000.npy + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: true + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: piqa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: hellaswag + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: winogrande + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: openbook_qa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: boolq + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: sciq + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: arc_easy + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: arc_challenge + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: copa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: commonsense_qa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: social_iqa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_stem_var + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_humanities_var + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_social_sciences_var + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_other_var + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_stem_mc_5shot + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_humanities_mc_5shot + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_social_sciences_mc_5shot + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_other_mc_5shot + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_stem_mc_5shot_test + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_humanities_mc_5shot_test + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_social_sciences_mc_5shot_test + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_other_mc_5shot_test + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +eval_interval: 2000 +tokenizer: + identifier: tokenizers/allenai_gpt-neox-olmo-dolma-v1_5.json + truncate_direction: right +save_folder: runs/olmoe-7b-29b-2ep +remote_save_folder: null +canceled_check_interval: 50 +save_interval: 2000 +save_interval_unsharded: null +save_interval_ephemeral: null +save_num_checkpoints_to_keep: 5 +save_num_unsharded_checkpoints_to_keep: -1 +save_overwrite: true +force_save_unsharded: false +no_pre_train_checkpoint: true +load_path: null +load_path_sharded_checkpointer: null +try_load_latest_save: false +reset_optimizer_state: false +reset_trainer_state: false +sharded_checkpointer: olmo_core +new_style_checkpoints: null +max_duration: 2ep +global_train_batch_size: 1024 +device_train_batch_size: 256 +device_train_microbatch_size: 2 +device_eval_batch_size: 1 +eval_subset_num_batches: -1 +eval_on_load: false +device_train_grad_accum: 128 +max_grad_norm: 1.0 +max_grad_norm_ratio: null +precision: amp_bf16 +wandb: + project: hiomoe + entity: bulgakov + group: 500gb-test + name: olmoe-7b-29b-2ep + tags: + - watching + log_artifacts: false + rank_zero_only: true + log_interval: 1 +speed_monitor: + window_size: 1 + gpu_flops_available: null +console_log_interval: 1 +gen1_gc_interval: 1 +compile: null +distributed_strategy: fsdp +fsdp: + use_orig_params: true + sharding_strategy: FULL_SHARD + wrapping_strategy: by_block + precision: mixed + hybrid_sharding_num_model_replicas: null +ddp: null +softmax_auxiliary_loss: false +auxiliary_loss_multiplier: 0.0001 +time_limit: null +extra_steps_after_cancel: 10 +early_stopping_factor: null +save_data_indices: true +python_profiling: false +torch_profiling: false +stop_at: null +stop_after: null +activation_checkpointing: fine_grained +fused_loss: null +hf_datasets_cache_dir: null +module_outputs_save_steps: null diff --git a/data-indices/rank0.tsv.gz b/data-indices/rank0.tsv.gz new file mode 100644 index 0000000000000000000000000000000000000000..51e1321f6e92daeb30610ab9cdd7371b7c8f6232 --- /dev/null +++ b/data-indices/rank0.tsv.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43c03869cd5c990f496bd28f24ac1dd9d8e0c0e23930e9700614ca9c0fc6ac3b +size 13343505 diff --git a/data-indices/rank2.tsv.gz b/data-indices/rank2.tsv.gz new file mode 100644 index 0000000000000000000000000000000000000000..7c32ef3ad28e570e17ed0fd3983cb22b3650e7e3 --- /dev/null +++ b/data-indices/rank2.tsv.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ce5ace18c2e62b52f7bcf6ef376f2f0e68828d01054c038bbad72add4e1ae24 +size 13342151 diff --git a/data-indices/rank3.tsv.gz b/data-indices/rank3.tsv.gz new file mode 100644 index 0000000000000000000000000000000000000000..0d178c59c738fb5c17d151b3127d0f272356cb09 --- /dev/null +++ b/data-indices/rank3.tsv.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c85d8a644470edb0afb2d9a3c6db110673cfc173476a7451601834d8ee9b04f +size 13343526 diff --git a/step10000/config.yaml b/step10000/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e79a34c76837f0912ef5bffaffa6443554839a06 --- /dev/null +++ b/step10000/config.yaml @@ -0,0 +1,724 @@ +run_name: olmoe-7b-29b-2ep +seed: 6198 +epoch: null +dry_run: false +model: + d_model: 2048 + n_heads: 16 + n_kv_heads: null + clip_qkv: null + n_layers: 16 + mlp_ratio: 1 + mlp_hidden_size: null + activation_type: swiglu + block_type: moe + block_group_size: 1 + alibi: false + alibi_bias_max: 8.0 + rope: true + rope_full_precision: true + rope_theta: 10000 + flash_attention: true + attention_dropout: 0.0 + multi_query_attention: null + attention_layer_norm: true + residual_dropout: 0.0 + embedding_dropout: 0.0 + embedding_layer_norm: false + layer_norm_type: rms + layer_norm_with_affine: true + layer_norm_eps: 1.0e-05 + attention_layer_norm_with_affine: true + max_sequence_length: 4096 + include_bias: false + bias_for_layer_norm: false + scale_logits: false + vocab_size: 50280 + embedding_size: 50304 + weight_tying: false + eos_token_id: 0 + pad_token_id: 1 + init_device: meta + init_fn: normal + init_std: 0.02 + init_cutoff_factor: 3.0 + precision: amp_bf16 + moe_num_experts: 64 + moe_top_k: 8 + moe_mlp_impl: sparse + moe_log_expert_assignment: false + moe_shared_expert: false + moe_lbl_in_fp32: false + moe_loss_weight: 0.01 + moe_zloss_weight: 0.001 + moe_dropless: false + moe_capacity_factor: 1.25 + himoe_enabled: false + himoe_n_groups: 4 + himoe_top_k_per_group: 2 + himoe_lambda1: 0.01 + himoe_lambda2: 0.1 + scale_emb_init: false + emb_init_std: null + norm_after: false +optimizer: + name: adamw + learning_rate: 0.0004 + weight_decay: 0.1 + betas: + - 0.9 + - 0.95 + eps: 1.0e-08 + no_decay_norm_and_bias: null + selective_updates: false + decay_norm_and_bias: true + decay_embeddings: true + metrics_log_interval: 10 + record_update_metrics: false +scheduler: + name: cosine_with_warmup + units: tokens + t_warmup: 1405296000 + t_max: 58540000000 + alpha_f: 0.1 + grad_clip_warmup_steps: null + grad_clip_warmup_factor: null + warmup_min_lr: null +data: + paths: + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-13-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-14-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-15-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-000-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-001-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-002-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-003-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-004-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-005-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-006-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-007-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-008-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-009-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-010-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-011-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-012-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-013-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 8 + drop_last: true + pin_memory: true + prefetch_factor: 8 + persistent_workers: true + timeout: 0 + seed: null + instance_filter: + repetition_max_period: 13 + repetition_min_period: 1 + repetition_max_count: 32 +restore_dataloader: true +fast_forward_batches: null +evaluators: +- label: all-small-ppl-validation + type: lm + data: + paths: null + memmap_dtype: uint16 + datasets: + c4_en-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/c4_en/val/part-0-00000.npy + dolma_books-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_books/val/part-0-00000.npy + dolma_common-crawl-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_common-crawl/val/part-0-00000.npy + dolma_pes2o-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_pes2o/val/part-0-00000.npy + dolma_reddit-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_reddit/val/part-0-00000.npy + dolma_stack-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_stack/val/part-0-00000.npy + dolma_wiki-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_wiki/val/part-0-00000.npy + ice-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/ice/val/part-0-00000.npy + m2d2_s2orc-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/m2d2_s2orc/val/part-0-00000.npy + pile-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/pile/val/part-0-00000.npy + wikitext_103-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/wikitext_103/val/part-0-00000.npy + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: true + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: piqa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: hellaswag + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: winogrande + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: openbook_qa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: boolq + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: sciq + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: arc_easy + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: arc_challenge + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: copa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: commonsense_qa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: social_iqa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_stem_var + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_humanities_var + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_social_sciences_var + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_other_var + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_stem_mc_5shot + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_humanities_mc_5shot + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_social_sciences_mc_5shot + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_other_mc_5shot + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_stem_mc_5shot_test + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_humanities_mc_5shot_test + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_social_sciences_mc_5shot_test + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_other_mc_5shot_test + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +eval_interval: 2000 +tokenizer: + identifier: tokenizers/allenai_gpt-neox-olmo-dolma-v1_5.json + truncate_direction: right +save_folder: runs/olmoe-7b-29b-2ep +remote_save_folder: null +canceled_check_interval: 50 +save_interval: 2000 +save_interval_unsharded: null +save_interval_ephemeral: null +save_num_checkpoints_to_keep: 5 +save_num_unsharded_checkpoints_to_keep: -1 +save_overwrite: true +force_save_unsharded: false +no_pre_train_checkpoint: true +load_path: null +load_path_sharded_checkpointer: null +try_load_latest_save: false +reset_optimizer_state: false +reset_trainer_state: false +sharded_checkpointer: olmo_core +new_style_checkpoints: null +max_duration: 2ep +global_train_batch_size: 1024 +device_train_batch_size: 256 +device_train_microbatch_size: 2 +device_eval_batch_size: 1 +eval_subset_num_batches: -1 +eval_on_load: false +device_train_grad_accum: 128 +max_grad_norm: 1.0 +max_grad_norm_ratio: null +precision: amp_bf16 +wandb: + project: hiomoe + entity: bulgakov + group: 500gb-test + name: olmoe-7b-29b-2ep + tags: + - watching + log_artifacts: false + rank_zero_only: true + log_interval: 1 +speed_monitor: + window_size: 1 + gpu_flops_available: null +console_log_interval: 1 +gen1_gc_interval: 1 +compile: null +distributed_strategy: fsdp +fsdp: + use_orig_params: true + sharding_strategy: FULL_SHARD + wrapping_strategy: by_block + precision: mixed + hybrid_sharding_num_model_replicas: null +ddp: null +softmax_auxiliary_loss: false +auxiliary_loss_multiplier: 0.0001 +time_limit: null +extra_steps_after_cancel: 10 +early_stopping_factor: null +save_data_indices: true +python_profiling: false +torch_profiling: false +stop_at: 13964 +stop_after: null +activation_checkpointing: fine_grained +fused_loss: null +hf_datasets_cache_dir: null +module_outputs_save_steps: null diff --git a/step10000/model/metadata.json b/step10000/model/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..7510f520c74f60640a475261f5500694e2ff4edd --- /dev/null +++ b/step10000/model/metadata.json @@ -0,0 +1 @@ +{"tensors": {"transformer.wte.weight": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 51511808]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[51511808, 103022592]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.ln_f.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 1024]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[1024, 2048]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.ff_out.weight": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 51510784]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[51510784, 103022592]], "local_shape": null, "global_offset": null}}}}} \ No newline at end of file diff --git a/step10000/optim/metadata.json b/step10000/optim/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..da4f797ae9bbc123d9a04a97676fed9e7a54884c --- /dev/null +++ b/step10000/optim/metadata.json @@ -0,0 +1 @@ +{"tensors": {"num_param_groups": {"shape": [], "is_sharded": false, "dtype": "I64", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "param_group0": {"shape": [18530], "is_sharded": false, "dtype": "U8", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 18530]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.wte.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.wte.weight.__exp_avg": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 51511808]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[51511808, 103022592]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.wte.weight.__exp_avg_sq": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 51511808]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[51511808, 103022592]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state_keys": {"shape": [46], "is_sharded": false, "dtype": "U8", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 46]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.ln_f.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.ln_f.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 1024]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[1024, 2048]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.ln_f.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 1024]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[1024, 2048]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.ff_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_2.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.ff_out.weight.__exp_avg": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 51510784]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[51510784, 103022592]], "local_shape": null, "global_offset": null}}}, "state.transformer.ff_out.weight.__exp_avg_sq": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 51510784]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[51510784, 103022592]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}}} \ No newline at end of file diff --git a/step10000/train/rank0.pt b/step10000/train/rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..dc6f29fe23c853516a69fa43b9d89d2eb549a8d2 --- /dev/null +++ b/step10000/train/rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:686920a2f9baff37b9093b8f45a6955f2803a3a05e09a6c5c3ea873c936927fe +size 14997 diff --git a/step10000/train/rank1.pt b/step10000/train/rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..16e7c3e74623d8080ef46f98db811ed4c8a7ce83 --- /dev/null +++ b/step10000/train/rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8447e2cc3ddfd7ab2a4d5b5bd59292ed81f72d899314916fc25bdbaed3d3add0 +size 14997 diff --git a/step10000/train/rank2.pt b/step10000/train/rank2.pt new file mode 100644 index 0000000000000000000000000000000000000000..5ce56d195f302e2e8dc14a5a7d8bbac83d0aeb4a --- /dev/null +++ b/step10000/train/rank2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36dfd18a006ae258a9b8ce1cfc74e9827208dfab2c7aa8dd7b4b42fcfb2f0753 +size 14997 diff --git a/step10000/train/rank3.pt b/step10000/train/rank3.pt new file mode 100644 index 0000000000000000000000000000000000000000..9d0afec474f97fb4d5934a70f0f8c8c4faec456a --- /dev/null +++ b/step10000/train/rank3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f16d62720f8ba4ba2cfdce457771cfa447cb32553ed323340bffc98a96407c3 +size 14997 diff --git a/step12000/config.yaml b/step12000/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e79a34c76837f0912ef5bffaffa6443554839a06 --- /dev/null +++ b/step12000/config.yaml @@ -0,0 +1,724 @@ +run_name: olmoe-7b-29b-2ep +seed: 6198 +epoch: null +dry_run: false +model: + d_model: 2048 + n_heads: 16 + n_kv_heads: null + clip_qkv: null + n_layers: 16 + mlp_ratio: 1 + mlp_hidden_size: null + activation_type: swiglu + block_type: moe + block_group_size: 1 + alibi: false + alibi_bias_max: 8.0 + rope: true + rope_full_precision: true + rope_theta: 10000 + flash_attention: true + attention_dropout: 0.0 + multi_query_attention: null + attention_layer_norm: true + residual_dropout: 0.0 + embedding_dropout: 0.0 + embedding_layer_norm: false + layer_norm_type: rms + layer_norm_with_affine: true + layer_norm_eps: 1.0e-05 + attention_layer_norm_with_affine: true + max_sequence_length: 4096 + include_bias: false + bias_for_layer_norm: false + scale_logits: false + vocab_size: 50280 + embedding_size: 50304 + weight_tying: false + eos_token_id: 0 + pad_token_id: 1 + init_device: meta + init_fn: normal + init_std: 0.02 + init_cutoff_factor: 3.0 + precision: amp_bf16 + moe_num_experts: 64 + moe_top_k: 8 + moe_mlp_impl: sparse + moe_log_expert_assignment: false + moe_shared_expert: false + moe_lbl_in_fp32: false + moe_loss_weight: 0.01 + moe_zloss_weight: 0.001 + moe_dropless: false + moe_capacity_factor: 1.25 + himoe_enabled: false + himoe_n_groups: 4 + himoe_top_k_per_group: 2 + himoe_lambda1: 0.01 + himoe_lambda2: 0.1 + scale_emb_init: false + emb_init_std: null + norm_after: false +optimizer: + name: adamw + learning_rate: 0.0004 + weight_decay: 0.1 + betas: + - 0.9 + - 0.95 + eps: 1.0e-08 + no_decay_norm_and_bias: null + selective_updates: false + decay_norm_and_bias: true + decay_embeddings: true + metrics_log_interval: 10 + record_update_metrics: false +scheduler: + name: cosine_with_warmup + units: tokens + t_warmup: 1405296000 + t_max: 58540000000 + alpha_f: 0.1 + grad_clip_warmup_steps: null + grad_clip_warmup_factor: null + warmup_min_lr: null +data: + paths: + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-13-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-14-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-15-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-000-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-001-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-002-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-003-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-004-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-005-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-006-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-007-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-008-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-009-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-010-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-011-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-012-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-013-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 8 + drop_last: true + pin_memory: true + prefetch_factor: 8 + persistent_workers: true + timeout: 0 + seed: null + instance_filter: + repetition_max_period: 13 + repetition_min_period: 1 + repetition_max_count: 32 +restore_dataloader: true +fast_forward_batches: null +evaluators: +- label: all-small-ppl-validation + type: lm + data: + paths: null + memmap_dtype: uint16 + datasets: + c4_en-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/c4_en/val/part-0-00000.npy + dolma_books-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_books/val/part-0-00000.npy + dolma_common-crawl-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_common-crawl/val/part-0-00000.npy + dolma_pes2o-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_pes2o/val/part-0-00000.npy + dolma_reddit-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_reddit/val/part-0-00000.npy + dolma_stack-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_stack/val/part-0-00000.npy + dolma_wiki-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_wiki/val/part-0-00000.npy + ice-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/ice/val/part-0-00000.npy + m2d2_s2orc-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/m2d2_s2orc/val/part-0-00000.npy + pile-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/pile/val/part-0-00000.npy + wikitext_103-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/wikitext_103/val/part-0-00000.npy + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: true + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: piqa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: hellaswag + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: winogrande + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: openbook_qa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: boolq + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: sciq + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: arc_easy + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: arc_challenge + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: copa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: commonsense_qa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: social_iqa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_stem_var + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_humanities_var + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_social_sciences_var + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_other_var + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_stem_mc_5shot + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_humanities_mc_5shot + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_social_sciences_mc_5shot + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_other_mc_5shot + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_stem_mc_5shot_test + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_humanities_mc_5shot_test + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_social_sciences_mc_5shot_test + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_other_mc_5shot_test + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +eval_interval: 2000 +tokenizer: + identifier: tokenizers/allenai_gpt-neox-olmo-dolma-v1_5.json + truncate_direction: right +save_folder: runs/olmoe-7b-29b-2ep +remote_save_folder: null +canceled_check_interval: 50 +save_interval: 2000 +save_interval_unsharded: null +save_interval_ephemeral: null +save_num_checkpoints_to_keep: 5 +save_num_unsharded_checkpoints_to_keep: -1 +save_overwrite: true +force_save_unsharded: false +no_pre_train_checkpoint: true +load_path: null +load_path_sharded_checkpointer: null +try_load_latest_save: false +reset_optimizer_state: false +reset_trainer_state: false +sharded_checkpointer: olmo_core +new_style_checkpoints: null +max_duration: 2ep +global_train_batch_size: 1024 +device_train_batch_size: 256 +device_train_microbatch_size: 2 +device_eval_batch_size: 1 +eval_subset_num_batches: -1 +eval_on_load: false +device_train_grad_accum: 128 +max_grad_norm: 1.0 +max_grad_norm_ratio: null +precision: amp_bf16 +wandb: + project: hiomoe + entity: bulgakov + group: 500gb-test + name: olmoe-7b-29b-2ep + tags: + - watching + log_artifacts: false + rank_zero_only: true + log_interval: 1 +speed_monitor: + window_size: 1 + gpu_flops_available: null +console_log_interval: 1 +gen1_gc_interval: 1 +compile: null +distributed_strategy: fsdp +fsdp: + use_orig_params: true + sharding_strategy: FULL_SHARD + wrapping_strategy: by_block + precision: mixed + hybrid_sharding_num_model_replicas: null +ddp: null +softmax_auxiliary_loss: false +auxiliary_loss_multiplier: 0.0001 +time_limit: null +extra_steps_after_cancel: 10 +early_stopping_factor: null +save_data_indices: true +python_profiling: false +torch_profiling: false +stop_at: 13964 +stop_after: null +activation_checkpointing: fine_grained +fused_loss: null +hf_datasets_cache_dir: null +module_outputs_save_steps: null diff --git a/step12000/model/metadata.json b/step12000/model/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..7510f520c74f60640a475261f5500694e2ff4edd --- /dev/null +++ b/step12000/model/metadata.json @@ -0,0 +1 @@ +{"tensors": {"transformer.wte.weight": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 51511808]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[51511808, 103022592]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.ln_f.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 1024]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[1024, 2048]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.ff_out.weight": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 51510784]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[51510784, 103022592]], "local_shape": null, "global_offset": null}}}}} \ No newline at end of file diff --git a/step12000/optim/metadata.json b/step12000/optim/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..da4f797ae9bbc123d9a04a97676fed9e7a54884c --- /dev/null +++ b/step12000/optim/metadata.json @@ -0,0 +1 @@ +{"tensors": {"num_param_groups": {"shape": [], "is_sharded": false, "dtype": "I64", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "param_group0": {"shape": [18530], "is_sharded": false, "dtype": "U8", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 18530]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.wte.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.wte.weight.__exp_avg": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 51511808]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[51511808, 103022592]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.wte.weight.__exp_avg_sq": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 51511808]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[51511808, 103022592]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state_keys": {"shape": [46], "is_sharded": false, "dtype": "U8", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 46]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.ln_f.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.ln_f.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 1024]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[1024, 2048]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.ln_f.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 1024]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[1024, 2048]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.ff_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_2.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.ff_out.weight.__exp_avg": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 51510784]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[51510784, 103022592]], "local_shape": null, "global_offset": null}}}, "state.transformer.ff_out.weight.__exp_avg_sq": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 51510784]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[51510784, 103022592]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}}} \ No newline at end of file diff --git a/step12000/train/rank0.pt b/step12000/train/rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..6b7e55f6d36d2ec24baee0534bf2516091eefc82 --- /dev/null +++ b/step12000/train/rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d87bab02ce17c12a45a78b586cbb9120d29e03127306bfc91bd6dc44bbcc6e1 +size 15061 diff --git a/step12000/train/rank1.pt b/step12000/train/rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..4deef883c17b18dc76752c6bda4b73542a1f48be --- /dev/null +++ b/step12000/train/rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bd04f1d6abcbb4002833ee6e46eef7fac65e44d625ab7b87eca22d82b242940 +size 15061 diff --git a/step12000/train/rank2.pt b/step12000/train/rank2.pt new file mode 100644 index 0000000000000000000000000000000000000000..235507309c9e0488d51b34199b553a28f59d5440 --- /dev/null +++ b/step12000/train/rank2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9406840828e8ffe5ec25a17d0b8c2a42ba439ee3cd08f7aa2e301d12a38f666f +size 15061 diff --git a/step12000/train/rank3.pt b/step12000/train/rank3.pt new file mode 100644 index 0000000000000000000000000000000000000000..48cba32ae099ee690f12b1a8558d536902609f7f --- /dev/null +++ b/step12000/train/rank3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf4f7989b941e7695d4d8989b9997e97791ca12658e880f8916c49d1c51bab90 +size 15061 diff --git a/step13954/config.yaml b/step13954/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e79a34c76837f0912ef5bffaffa6443554839a06 --- /dev/null +++ b/step13954/config.yaml @@ -0,0 +1,724 @@ +run_name: olmoe-7b-29b-2ep +seed: 6198 +epoch: null +dry_run: false +model: + d_model: 2048 + n_heads: 16 + n_kv_heads: null + clip_qkv: null + n_layers: 16 + mlp_ratio: 1 + mlp_hidden_size: null + activation_type: swiglu + block_type: moe + block_group_size: 1 + alibi: false + alibi_bias_max: 8.0 + rope: true + rope_full_precision: true + rope_theta: 10000 + flash_attention: true + attention_dropout: 0.0 + multi_query_attention: null + attention_layer_norm: true + residual_dropout: 0.0 + embedding_dropout: 0.0 + embedding_layer_norm: false + layer_norm_type: rms + layer_norm_with_affine: true + layer_norm_eps: 1.0e-05 + attention_layer_norm_with_affine: true + max_sequence_length: 4096 + include_bias: false + bias_for_layer_norm: false + scale_logits: false + vocab_size: 50280 + embedding_size: 50304 + weight_tying: false + eos_token_id: 0 + pad_token_id: 1 + init_device: meta + init_fn: normal + init_std: 0.02 + init_cutoff_factor: 3.0 + precision: amp_bf16 + moe_num_experts: 64 + moe_top_k: 8 + moe_mlp_impl: sparse + moe_log_expert_assignment: false + moe_shared_expert: false + moe_lbl_in_fp32: false + moe_loss_weight: 0.01 + moe_zloss_weight: 0.001 + moe_dropless: false + moe_capacity_factor: 1.25 + himoe_enabled: false + himoe_n_groups: 4 + himoe_top_k_per_group: 2 + himoe_lambda1: 0.01 + himoe_lambda2: 0.1 + scale_emb_init: false + emb_init_std: null + norm_after: false +optimizer: + name: adamw + learning_rate: 0.0004 + weight_decay: 0.1 + betas: + - 0.9 + - 0.95 + eps: 1.0e-08 + no_decay_norm_and_bias: null + selective_updates: false + decay_norm_and_bias: true + decay_embeddings: true + metrics_log_interval: 10 + record_update_metrics: false +scheduler: + name: cosine_with_warmup + units: tokens + t_warmup: 1405296000 + t_max: 58540000000 + alpha_f: 0.1 + grad_clip_warmup_steps: null + grad_clip_warmup_factor: null + warmup_min_lr: null +data: + paths: + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-13-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-14-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-15-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-000-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-001-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-002-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-003-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-004-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-005-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-006-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-007-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-008-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-009-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-010-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-011-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-012-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-013-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 8 + drop_last: true + pin_memory: true + prefetch_factor: 8 + persistent_workers: true + timeout: 0 + seed: null + instance_filter: + repetition_max_period: 13 + repetition_min_period: 1 + repetition_max_count: 32 +restore_dataloader: true +fast_forward_batches: null +evaluators: +- label: all-small-ppl-validation + type: lm + data: + paths: null + memmap_dtype: uint16 + datasets: + c4_en-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/c4_en/val/part-0-00000.npy + dolma_books-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_books/val/part-0-00000.npy + dolma_common-crawl-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_common-crawl/val/part-0-00000.npy + dolma_pes2o-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_pes2o/val/part-0-00000.npy + dolma_reddit-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_reddit/val/part-0-00000.npy + dolma_stack-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_stack/val/part-0-00000.npy + dolma_wiki-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_wiki/val/part-0-00000.npy + ice-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/ice/val/part-0-00000.npy + m2d2_s2orc-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/m2d2_s2orc/val/part-0-00000.npy + pile-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/pile/val/part-0-00000.npy + wikitext_103-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/wikitext_103/val/part-0-00000.npy + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: true + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: piqa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: hellaswag + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: winogrande + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: openbook_qa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: boolq + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: sciq + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: arc_easy + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: arc_challenge + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: copa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: commonsense_qa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: social_iqa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_stem_var + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_humanities_var + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_social_sciences_var + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_other_var + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_stem_mc_5shot + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_humanities_mc_5shot + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_social_sciences_mc_5shot + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_other_mc_5shot + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_stem_mc_5shot_test + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_humanities_mc_5shot_test + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_social_sciences_mc_5shot_test + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_other_mc_5shot_test + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +eval_interval: 2000 +tokenizer: + identifier: tokenizers/allenai_gpt-neox-olmo-dolma-v1_5.json + truncate_direction: right +save_folder: runs/olmoe-7b-29b-2ep +remote_save_folder: null +canceled_check_interval: 50 +save_interval: 2000 +save_interval_unsharded: null +save_interval_ephemeral: null +save_num_checkpoints_to_keep: 5 +save_num_unsharded_checkpoints_to_keep: -1 +save_overwrite: true +force_save_unsharded: false +no_pre_train_checkpoint: true +load_path: null +load_path_sharded_checkpointer: null +try_load_latest_save: false +reset_optimizer_state: false +reset_trainer_state: false +sharded_checkpointer: olmo_core +new_style_checkpoints: null +max_duration: 2ep +global_train_batch_size: 1024 +device_train_batch_size: 256 +device_train_microbatch_size: 2 +device_eval_batch_size: 1 +eval_subset_num_batches: -1 +eval_on_load: false +device_train_grad_accum: 128 +max_grad_norm: 1.0 +max_grad_norm_ratio: null +precision: amp_bf16 +wandb: + project: hiomoe + entity: bulgakov + group: 500gb-test + name: olmoe-7b-29b-2ep + tags: + - watching + log_artifacts: false + rank_zero_only: true + log_interval: 1 +speed_monitor: + window_size: 1 + gpu_flops_available: null +console_log_interval: 1 +gen1_gc_interval: 1 +compile: null +distributed_strategy: fsdp +fsdp: + use_orig_params: true + sharding_strategy: FULL_SHARD + wrapping_strategy: by_block + precision: mixed + hybrid_sharding_num_model_replicas: null +ddp: null +softmax_auxiliary_loss: false +auxiliary_loss_multiplier: 0.0001 +time_limit: null +extra_steps_after_cancel: 10 +early_stopping_factor: null +save_data_indices: true +python_profiling: false +torch_profiling: false +stop_at: 13964 +stop_after: null +activation_checkpointing: fine_grained +fused_loss: null +hf_datasets_cache_dir: null +module_outputs_save_steps: null diff --git a/step13954/model/metadata.json b/step13954/model/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..7510f520c74f60640a475261f5500694e2ff4edd --- /dev/null +++ b/step13954/model/metadata.json @@ -0,0 +1 @@ +{"tensors": {"transformer.wte.weight": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 51511808]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[51511808, 103022592]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.ln_f.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 1024]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[1024, 2048]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.ff_out.weight": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 51510784]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[51510784, 103022592]], "local_shape": null, "global_offset": null}}}}} \ No newline at end of file diff --git a/step13954/optim/metadata.json b/step13954/optim/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..da4f797ae9bbc123d9a04a97676fed9e7a54884c --- /dev/null +++ b/step13954/optim/metadata.json @@ -0,0 +1 @@ +{"tensors": {"num_param_groups": {"shape": [], "is_sharded": false, "dtype": "I64", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "param_group0": {"shape": [18530], "is_sharded": false, "dtype": "U8", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 18530]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.wte.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.wte.weight.__exp_avg": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 51511808]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[51511808, 103022592]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.wte.weight.__exp_avg_sq": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 51511808]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[51511808, 103022592]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state_keys": {"shape": [46], "is_sharded": false, "dtype": "U8", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 46]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.ln_f.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.ln_f.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 1024]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[1024, 2048]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.ln_f.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 1024]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[1024, 2048]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.ff_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_2.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.ff_out.weight.__exp_avg": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 51510784]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[51510784, 103022592]], "local_shape": null, "global_offset": null}}}, "state.transformer.ff_out.weight.__exp_avg_sq": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 51510784]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[51510784, 103022592]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}}} \ No newline at end of file diff --git a/step13954/train/rank0.pt b/step13954/train/rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..015c7443ff0abb26bc181a94b29be37c7361be2a --- /dev/null +++ b/step13954/train/rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0132cb0d342bbcb6da79c45648ac7a87b83ba054aed6192faf46190f9d03b41 +size 15061 diff --git a/step13954/train/rank1.pt b/step13954/train/rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..24e72f6b3898e7465f1f59b12c0df8f2fa88cd2f --- /dev/null +++ b/step13954/train/rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9070989c555d4e0195bcbb03e97b71296fadff0ed280c77758bf7eff7fe100d7 +size 15061 diff --git a/step13954/train/rank2.pt b/step13954/train/rank2.pt new file mode 100644 index 0000000000000000000000000000000000000000..24f55507d590c42d1ca9630785911d20f1ea06d6 --- /dev/null +++ b/step13954/train/rank2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d52aaa729bcdd1ca6b2b3cfd700c1eb99f2b7984f5db75ecac2c15715b18f0e6 +size 15061 diff --git a/step13954/train/rank3.pt b/step13954/train/rank3.pt new file mode 100644 index 0000000000000000000000000000000000000000..e2e6b23944b1671281b0523cabdcdc7bcda7a942 --- /dev/null +++ b/step13954/train/rank3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0be30cfbd412cee2d35d4e71acd9e5cd11df24c02b5a81b30d97cdfc6ce757e4 +size 15061 diff --git a/step6000/config.yaml b/step6000/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e79a34c76837f0912ef5bffaffa6443554839a06 --- /dev/null +++ b/step6000/config.yaml @@ -0,0 +1,724 @@ +run_name: olmoe-7b-29b-2ep +seed: 6198 +epoch: null +dry_run: false +model: + d_model: 2048 + n_heads: 16 + n_kv_heads: null + clip_qkv: null + n_layers: 16 + mlp_ratio: 1 + mlp_hidden_size: null + activation_type: swiglu + block_type: moe + block_group_size: 1 + alibi: false + alibi_bias_max: 8.0 + rope: true + rope_full_precision: true + rope_theta: 10000 + flash_attention: true + attention_dropout: 0.0 + multi_query_attention: null + attention_layer_norm: true + residual_dropout: 0.0 + embedding_dropout: 0.0 + embedding_layer_norm: false + layer_norm_type: rms + layer_norm_with_affine: true + layer_norm_eps: 1.0e-05 + attention_layer_norm_with_affine: true + max_sequence_length: 4096 + include_bias: false + bias_for_layer_norm: false + scale_logits: false + vocab_size: 50280 + embedding_size: 50304 + weight_tying: false + eos_token_id: 0 + pad_token_id: 1 + init_device: meta + init_fn: normal + init_std: 0.02 + init_cutoff_factor: 3.0 + precision: amp_bf16 + moe_num_experts: 64 + moe_top_k: 8 + moe_mlp_impl: sparse + moe_log_expert_assignment: false + moe_shared_expert: false + moe_lbl_in_fp32: false + moe_loss_weight: 0.01 + moe_zloss_weight: 0.001 + moe_dropless: false + moe_capacity_factor: 1.25 + himoe_enabled: false + himoe_n_groups: 4 + himoe_top_k_per_group: 2 + himoe_lambda1: 0.01 + himoe_lambda2: 0.1 + scale_emb_init: false + emb_init_std: null + norm_after: false +optimizer: + name: adamw + learning_rate: 0.0004 + weight_decay: 0.1 + betas: + - 0.9 + - 0.95 + eps: 1.0e-08 + no_decay_norm_and_bias: null + selective_updates: false + decay_norm_and_bias: true + decay_embeddings: true + metrics_log_interval: 10 + record_update_metrics: false +scheduler: + name: cosine_with_warmup + units: tokens + t_warmup: 1405296000 + t_max: 58540000000 + alpha_f: 0.1 + grad_clip_warmup_steps: null + grad_clip_warmup_factor: null + warmup_min_lr: null +data: + paths: + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-13-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-14-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-15-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-000-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-001-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-002-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-003-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-004-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-005-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-006-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-007-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-008-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-009-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-010-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-011-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-012-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-013-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 8 + drop_last: true + pin_memory: true + prefetch_factor: 8 + persistent_workers: true + timeout: 0 + seed: null + instance_filter: + repetition_max_period: 13 + repetition_min_period: 1 + repetition_max_count: 32 +restore_dataloader: true +fast_forward_batches: null +evaluators: +- label: all-small-ppl-validation + type: lm + data: + paths: null + memmap_dtype: uint16 + datasets: + c4_en-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/c4_en/val/part-0-00000.npy + dolma_books-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_books/val/part-0-00000.npy + dolma_common-crawl-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_common-crawl/val/part-0-00000.npy + dolma_pes2o-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_pes2o/val/part-0-00000.npy + dolma_reddit-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_reddit/val/part-0-00000.npy + dolma_stack-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_stack/val/part-0-00000.npy + dolma_wiki-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_wiki/val/part-0-00000.npy + ice-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/ice/val/part-0-00000.npy + m2d2_s2orc-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/m2d2_s2orc/val/part-0-00000.npy + pile-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/pile/val/part-0-00000.npy + wikitext_103-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/wikitext_103/val/part-0-00000.npy + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: true + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: piqa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: hellaswag + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: winogrande + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: openbook_qa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: boolq + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: sciq + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: arc_easy + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: arc_challenge + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: copa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: commonsense_qa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: social_iqa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_stem_var + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_humanities_var + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_social_sciences_var + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_other_var + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_stem_mc_5shot + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_humanities_mc_5shot + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_social_sciences_mc_5shot + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_other_mc_5shot + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_stem_mc_5shot_test + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_humanities_mc_5shot_test + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_social_sciences_mc_5shot_test + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_other_mc_5shot_test + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +eval_interval: 2000 +tokenizer: + identifier: tokenizers/allenai_gpt-neox-olmo-dolma-v1_5.json + truncate_direction: right +save_folder: runs/olmoe-7b-29b-2ep +remote_save_folder: null +canceled_check_interval: 50 +save_interval: 2000 +save_interval_unsharded: null +save_interval_ephemeral: null +save_num_checkpoints_to_keep: 5 +save_num_unsharded_checkpoints_to_keep: -1 +save_overwrite: true +force_save_unsharded: false +no_pre_train_checkpoint: true +load_path: null +load_path_sharded_checkpointer: null +try_load_latest_save: false +reset_optimizer_state: false +reset_trainer_state: false +sharded_checkpointer: olmo_core +new_style_checkpoints: null +max_duration: 2ep +global_train_batch_size: 1024 +device_train_batch_size: 256 +device_train_microbatch_size: 2 +device_eval_batch_size: 1 +eval_subset_num_batches: -1 +eval_on_load: false +device_train_grad_accum: 128 +max_grad_norm: 1.0 +max_grad_norm_ratio: null +precision: amp_bf16 +wandb: + project: hiomoe + entity: bulgakov + group: 500gb-test + name: olmoe-7b-29b-2ep + tags: + - watching + log_artifacts: false + rank_zero_only: true + log_interval: 1 +speed_monitor: + window_size: 1 + gpu_flops_available: null +console_log_interval: 1 +gen1_gc_interval: 1 +compile: null +distributed_strategy: fsdp +fsdp: + use_orig_params: true + sharding_strategy: FULL_SHARD + wrapping_strategy: by_block + precision: mixed + hybrid_sharding_num_model_replicas: null +ddp: null +softmax_auxiliary_loss: false +auxiliary_loss_multiplier: 0.0001 +time_limit: null +extra_steps_after_cancel: 10 +early_stopping_factor: null +save_data_indices: true +python_profiling: false +torch_profiling: false +stop_at: 13964 +stop_after: null +activation_checkpointing: fine_grained +fused_loss: null +hf_datasets_cache_dir: null +module_outputs_save_steps: null diff --git a/step6000/model/metadata.json b/step6000/model/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..7510f520c74f60640a475261f5500694e2ff4edd --- /dev/null +++ b/step6000/model/metadata.json @@ -0,0 +1 @@ +{"tensors": {"transformer.wte.weight": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 51511808]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[51511808, 103022592]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.ln_f.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 1024]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[1024, 2048]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.ff_out.weight": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 51510784]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[51510784, 103022592]], "local_shape": null, "global_offset": null}}}}} \ No newline at end of file diff --git a/step6000/optim/metadata.json b/step6000/optim/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..da4f797ae9bbc123d9a04a97676fed9e7a54884c --- /dev/null +++ b/step6000/optim/metadata.json @@ -0,0 +1 @@ +{"tensors": {"num_param_groups": {"shape": [], "is_sharded": false, "dtype": "I64", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "param_group0": {"shape": [18530], "is_sharded": false, "dtype": "U8", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 18530]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.wte.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.wte.weight.__exp_avg": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 51511808]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[51511808, 103022592]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.wte.weight.__exp_avg_sq": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 51511808]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[51511808, 103022592]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state_keys": {"shape": [46], "is_sharded": false, "dtype": "U8", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 46]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.ln_f.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.ln_f.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 1024]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[1024, 2048]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.ln_f.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 1024]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[1024, 2048]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.ff_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_2.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.ff_out.weight.__exp_avg": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 51510784]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[51510784, 103022592]], "local_shape": null, "global_offset": null}}}, "state.transformer.ff_out.weight.__exp_avg_sq": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 51510784]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[51510784, 103022592]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}}} \ No newline at end of file diff --git a/step6000/train/rank0.pt b/step6000/train/rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..8cad36bc4ccc7d02401866a5ae9299bdf57e9e59 --- /dev/null +++ b/step6000/train/rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c413c99a6338e5cb932add36c4070b4eecbe52fa500afa3729cc694c10f2dfbe +size 14933 diff --git a/step6000/train/rank1.pt b/step6000/train/rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..ea8592a19ebb5bce672ceab5a5a2aa89f47ae3bc --- /dev/null +++ b/step6000/train/rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee5f375854ccf6d5598d3fb3cb6928ca66af8b3da0f3f187a586256470d63c06 +size 14933 diff --git a/step6000/train/rank2.pt b/step6000/train/rank2.pt new file mode 100644 index 0000000000000000000000000000000000000000..3d3f2e248dc0c67aa5cf1286376a0cd1b89aa9fe --- /dev/null +++ b/step6000/train/rank2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba353b8a18abd862774ef6f1b0ba39e34dd4b6609305b8d8618374922418ef32 +size 14933 diff --git a/step6000/train/rank3.pt b/step6000/train/rank3.pt new file mode 100644 index 0000000000000000000000000000000000000000..5d5cf0bd30c96f527cd0fd775d9b75f5567ecf55 --- /dev/null +++ b/step6000/train/rank3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98002e33db5c87ed9e1d7a20d657971aa4046435183ff43ae38cd434e93a92d1 +size 14933 diff --git a/step8000/config.yaml b/step8000/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e79a34c76837f0912ef5bffaffa6443554839a06 --- /dev/null +++ b/step8000/config.yaml @@ -0,0 +1,724 @@ +run_name: olmoe-7b-29b-2ep +seed: 6198 +epoch: null +dry_run: false +model: + d_model: 2048 + n_heads: 16 + n_kv_heads: null + clip_qkv: null + n_layers: 16 + mlp_ratio: 1 + mlp_hidden_size: null + activation_type: swiglu + block_type: moe + block_group_size: 1 + alibi: false + alibi_bias_max: 8.0 + rope: true + rope_full_precision: true + rope_theta: 10000 + flash_attention: true + attention_dropout: 0.0 + multi_query_attention: null + attention_layer_norm: true + residual_dropout: 0.0 + embedding_dropout: 0.0 + embedding_layer_norm: false + layer_norm_type: rms + layer_norm_with_affine: true + layer_norm_eps: 1.0e-05 + attention_layer_norm_with_affine: true + max_sequence_length: 4096 + include_bias: false + bias_for_layer_norm: false + scale_logits: false + vocab_size: 50280 + embedding_size: 50304 + weight_tying: false + eos_token_id: 0 + pad_token_id: 1 + init_device: meta + init_fn: normal + init_std: 0.02 + init_cutoff_factor: 3.0 + precision: amp_bf16 + moe_num_experts: 64 + moe_top_k: 8 + moe_mlp_impl: sparse + moe_log_expert_assignment: false + moe_shared_expert: false + moe_lbl_in_fp32: false + moe_loss_weight: 0.01 + moe_zloss_weight: 0.001 + moe_dropless: false + moe_capacity_factor: 1.25 + himoe_enabled: false + himoe_n_groups: 4 + himoe_top_k_per_group: 2 + himoe_lambda1: 0.01 + himoe_lambda2: 0.1 + scale_emb_init: false + emb_init_std: null + norm_after: false +optimizer: + name: adamw + learning_rate: 0.0004 + weight_decay: 0.1 + betas: + - 0.9 + - 0.95 + eps: 1.0e-08 + no_decay_norm_and_bias: null + selective_updates: false + decay_norm_and_bias: true + decay_embeddings: true + metrics_log_interval: 10 + record_update_metrics: false +scheduler: + name: cosine_with_warmup + units: tokens + t_warmup: 1405296000 + t_max: 58540000000 + alpha_f: 0.1 + grad_clip_warmup_steps: null + grad_clip_warmup_factor: null + warmup_min_lr: null +data: + paths: + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-13-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-14-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-15-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-000-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-001-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-002-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-003-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-004-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-005-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-006-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-007-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-008-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-009-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-010-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-011-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-012-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-013-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 8 + drop_last: true + pin_memory: true + prefetch_factor: 8 + persistent_workers: true + timeout: 0 + seed: null + instance_filter: + repetition_max_period: 13 + repetition_min_period: 1 + repetition_max_count: 32 +restore_dataloader: true +fast_forward_batches: null +evaluators: +- label: all-small-ppl-validation + type: lm + data: + paths: null + memmap_dtype: uint16 + datasets: + c4_en-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/c4_en/val/part-0-00000.npy + dolma_books-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_books/val/part-0-00000.npy + dolma_common-crawl-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_common-crawl/val/part-0-00000.npy + dolma_pes2o-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_pes2o/val/part-0-00000.npy + dolma_reddit-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_reddit/val/part-0-00000.npy + dolma_stack-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_stack/val/part-0-00000.npy + dolma_wiki-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_wiki/val/part-0-00000.npy + ice-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/ice/val/part-0-00000.npy + m2d2_s2orc-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/m2d2_s2orc/val/part-0-00000.npy + pile-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/pile/val/part-0-00000.npy + wikitext_103-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/wikitext_103/val/part-0-00000.npy + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: true + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: piqa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: hellaswag + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: winogrande + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: openbook_qa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: boolq + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: sciq + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: arc_easy + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: arc_challenge + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: copa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: commonsense_qa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: social_iqa + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_stem_var + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_humanities_var + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_social_sciences_var + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_other_var + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_stem_mc_5shot + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_humanities_mc_5shot + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_social_sciences_mc_5shot + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_other_mc_5shot + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_stem_mc_5shot_test + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_humanities_mc_5shot_test + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_social_sciences_mc_5shot_test + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +- label: mmlu_other_mc_5shot_test + type: downstream + data: + paths: null + memmap_dtype: uint16 + datasets: null + label_mask_paths: null + pad_direction: right + generate_attention_mask: false + generate_doc_lengths: false + num_workers: 0 + drop_last: false + pin_memory: false + prefetch_factor: null + persistent_workers: false + timeout: 0 + seed: null + instance_filter: null + device_eval_batch_size: null + subset_num_batches: null +eval_interval: 2000 +tokenizer: + identifier: tokenizers/allenai_gpt-neox-olmo-dolma-v1_5.json + truncate_direction: right +save_folder: runs/olmoe-7b-29b-2ep +remote_save_folder: null +canceled_check_interval: 50 +save_interval: 2000 +save_interval_unsharded: null +save_interval_ephemeral: null +save_num_checkpoints_to_keep: 5 +save_num_unsharded_checkpoints_to_keep: -1 +save_overwrite: true +force_save_unsharded: false +no_pre_train_checkpoint: true +load_path: null +load_path_sharded_checkpointer: null +try_load_latest_save: false +reset_optimizer_state: false +reset_trainer_state: false +sharded_checkpointer: olmo_core +new_style_checkpoints: null +max_duration: 2ep +global_train_batch_size: 1024 +device_train_batch_size: 256 +device_train_microbatch_size: 2 +device_eval_batch_size: 1 +eval_subset_num_batches: -1 +eval_on_load: false +device_train_grad_accum: 128 +max_grad_norm: 1.0 +max_grad_norm_ratio: null +precision: amp_bf16 +wandb: + project: hiomoe + entity: bulgakov + group: 500gb-test + name: olmoe-7b-29b-2ep + tags: + - watching + log_artifacts: false + rank_zero_only: true + log_interval: 1 +speed_monitor: + window_size: 1 + gpu_flops_available: null +console_log_interval: 1 +gen1_gc_interval: 1 +compile: null +distributed_strategy: fsdp +fsdp: + use_orig_params: true + sharding_strategy: FULL_SHARD + wrapping_strategy: by_block + precision: mixed + hybrid_sharding_num_model_replicas: null +ddp: null +softmax_auxiliary_loss: false +auxiliary_loss_multiplier: 0.0001 +time_limit: null +extra_steps_after_cancel: 10 +early_stopping_factor: null +save_data_indices: true +python_profiling: false +torch_profiling: false +stop_at: 13964 +stop_after: null +activation_checkpointing: fine_grained +fused_loss: null +hf_datasets_cache_dir: null +module_outputs_save_steps: null diff --git a/step8000/model/metadata.json b/step8000/model/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..7510f520c74f60640a475261f5500694e2ff4edd --- /dev/null +++ b/step8000/model/metadata.json @@ -0,0 +1 @@ +{"tensors": {"transformer.wte.weight": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 51511808]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[51511808, 103022592]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.ln_f.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 1024]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[1024, 2048]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.0.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.1.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.2.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.3.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.4.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.5.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.6.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.7.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.8.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.9.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.10.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.11.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.12.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.13.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.14.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.k_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.q_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.attn_out.weight": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.router.layer.weight": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.experts.mlp.w1": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ffn.experts.mlp.w2": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.attn_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.ff_norm.weight": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "transformer.blocks.15.att_proj.weight": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "transformer.ff_out.weight": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 51510784]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[51510784, 103022592]], "local_shape": null, "global_offset": null}}}}} \ No newline at end of file diff --git a/step8000/optim/metadata.json b/step8000/optim/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..da4f797ae9bbc123d9a04a97676fed9e7a54884c --- /dev/null +++ b/step8000/optim/metadata.json @@ -0,0 +1 @@ +{"tensors": {"num_param_groups": {"shape": [], "is_sharded": false, "dtype": "I64", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "param_group0": {"shape": [18530], "is_sharded": false, "dtype": "U8", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 18530]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_out.weight.__exp_avg": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_out.weight.__exp_avg_sq": {"shape": [2048, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 4194304]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w1.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w1.__exp_avg": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w1.__exp_avg_sq": {"shape": [64, 2048, 1024], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 67008512]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[67008512, 134217728]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.router.layer.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.router.layer.weight.__exp_avg": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.router.layer.weight.__exp_avg_sq": {"shape": [64, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 131072]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.k_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.k_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.k_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.q_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.q_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.q_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.wte.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.wte.weight.__exp_avg": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 51511808]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[51511808, 103022592]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.wte.weight.__exp_avg_sq": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 51511808]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[51511808, 103022592]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state_keys": {"shape": [46], "is_sharded": false, "dtype": "U8", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 46]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w2.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w2.__exp_avg": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ffn.experts.mlp.w2.__exp_avg_sq": {"shape": [64, 1024, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 4128768]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[4128768, 75466752]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[75466752, 134217728]], "local_shape": null, "global_offset": null}}}, "state.transformer.ln_f.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_1.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.ln_f.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 1024]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[1024, 2048]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.ln_f.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 1024]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[1024, 2048]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}}}, "state.transformer.ff_out.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_2.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.ff_out.weight.__exp_avg": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 51510784]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[51510784, 103022592]], "local_shape": null, "global_offset": null}}}, "state.transformer.ff_out.weight.__exp_avg_sq": {"shape": [50304, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 51510784]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[51510784, 103022592]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.0.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.1.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.10.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.11.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.12.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.13.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.14.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.15.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.2.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.3.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.4.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.5.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.6.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.7.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.8.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.att_proj.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.att_proj.weight.__exp_avg": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.att_proj.weight.__exp_avg_sq": {"shape": [6144, 2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 12582912]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.attn_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ff_norm.weight.__step": {"shape": [], "is_sharded": false, "dtype": "F32", "shard_spec_per_file": {"rank_3.safetensors": {"flattened_offsets": [[0, 1]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ff_norm.weight.__exp_avg": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}, "state.transformer.blocks.9.ff_norm.weight.__exp_avg_sq": {"shape": [2048], "is_sharded": true, "dtype": "F32", "shard_spec_per_file": {"rank_0.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_1.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_2.safetensors": {"flattened_offsets": [[0, 0]], "local_shape": null, "global_offset": null}, "rank_3.safetensors": {"flattened_offsets": [[0, 2048]], "local_shape": null, "global_offset": null}}}}} \ No newline at end of file diff --git a/step8000/train/rank0.pt b/step8000/train/rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..9b87d88d2ad8120bfbe83ecc7c27c0e9187fedd7 --- /dev/null +++ b/step8000/train/rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55ce849fb5cf4a8f13e62bbe6888f4296d535f18938efc7de53bd7b7e4c3d84d +size 14997 diff --git a/step8000/train/rank1.pt b/step8000/train/rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..16873428bc8f0121d0aadf0968a3b18a8e34803c --- /dev/null +++ b/step8000/train/rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c6e8b4c1f4079dd71c5cd3c8b8912f7023501542fdfefb7db5d360bbdbbf6dc +size 14997 diff --git a/step8000/train/rank2.pt b/step8000/train/rank2.pt new file mode 100644 index 0000000000000000000000000000000000000000..2bc4566e3d726c468cb200c12c8da5aaba7516b0 --- /dev/null +++ b/step8000/train/rank2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abecbc4654726d4e171f18c7c70c809e8e61a87979e8d1adc88fde466ab9050b +size 14997 diff --git a/step8000/train/rank3.pt b/step8000/train/rank3.pt new file mode 100644 index 0000000000000000000000000000000000000000..90e43e326bab86c7f529d7b985013817e2d427c6 --- /dev/null +++ b/step8000/train/rank3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9485998d23382706ca787f8ef110b0ac28ad142a8429f9efbb4b5c12dc2a774c +size 14997 diff --git a/train_data/global_indices.npy b/train_data/global_indices.npy new file mode 100644 index 0000000000000000000000000000000000000000..c4b29edf0eac07a7622b0eec570cb5e8867d25ee --- /dev/null +++ b/train_data/global_indices.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:722c0fcfa8661ed55afcc890b069ffcfbd4c0e80cf26b3a8f282536bbe3cfd19 +size 28581312 diff --git a/wandb/wandb/debug-internal.log b/wandb/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..186f99d7e572e5cdbb6ee8916298db39d1ae396b --- /dev/null +++ b/wandb/wandb/debug-internal.log @@ -0,0 +1,13 @@ +{"time":"2025-12-19T03:28:09.715068381Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"} +{"time":"2025-12-19T03:28:10.010671632Z","level":"INFO","msg":"stream: created new stream","id":"fhag09gx"} +{"time":"2025-12-19T03:28:10.01079177Z","level":"INFO","msg":"writer: started","stream_id":"fhag09gx"} +{"time":"2025-12-19T03:28:10.010792247Z","level":"INFO","msg":"sender: started","stream_id":"fhag09gx"} +{"time":"2025-12-19T03:28:10.011088687Z","level":"INFO","msg":"handler: started","stream_id":"fhag09gx"} +{"time":"2025-12-19T03:28:10.011242775Z","level":"INFO","msg":"stream: started","id":"fhag09gx"} +{"time":"2025-12-19T03:28:10.069884068Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2025-12-25T13:58:15.121256706Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-12-25T13:58:15.388114558Z","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2025-12-25T13:58:15.404292716Z","level":"INFO","msg":"stream: closing","id":"fhag09gx"} +{"time":"2025-12-25T13:58:15.404311317Z","level":"INFO","msg":"handler: closed","stream_id":"fhag09gx"} +{"time":"2025-12-25T13:58:15.404690567Z","level":"INFO","msg":"sender: closed","stream_id":"fhag09gx"} +{"time":"2025-12-25T13:58:15.404702005Z","level":"INFO","msg":"stream: closed","id":"fhag09gx"} diff --git a/wandb/wandb/debug.log b/wandb/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..66e1e967bc1e3af2dfc513dfcb07874b1410f45f --- /dev/null +++ b/wandb/wandb/debug.log @@ -0,0 +1,26 @@ +2025-12-19 03:28:09,456 INFO MainThread:1675715 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1 +2025-12-19 03:28:09,456 INFO MainThread:1675715 [wandb_setup.py:_flush():80] Configure stats pid to 1675715 +2025-12-19 03:28:09,456 INFO MainThread:1675715 [wandb_setup.py:_flush():80] Loading settings from /data/users/xandi281/.config/wandb/settings +2025-12-19 03:28:09,456 INFO MainThread:1675715 [wandb_setup.py:_flush():80] Loading settings from /data/users/xandi281/work/Hi-OLMoE/OLMo/wandb/settings +2025-12-19 03:28:09,456 INFO MainThread:1675715 [wandb_setup.py:_flush():80] Loading settings from environment variables +2025-12-19 03:28:09,456 INFO MainThread:1675715 [wandb_init.py:setup_run_log_directory():714] Logging user logs to runs/olmoe-7b-29b-2ep/wandb/wandb/run-20251219_032809-fhag09gx/logs/debug.log +2025-12-19 03:28:09,456 INFO MainThread:1675715 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to runs/olmoe-7b-29b-2ep/wandb/wandb/run-20251219_032809-fhag09gx/logs/debug-internal.log +2025-12-19 03:28:09,456 INFO MainThread:1675715 [wandb_init.py:init():841] calling init triggers +2025-12-19 03:28:09,457 INFO MainThread:1675715 [wandb_init.py:init():846] wandb.init called with sweep_config: {} +config: {'run_name': 'olmoe-7b-29b-2ep', 'seed': 6198, 'epoch': None, 'dry_run': False, 'model': {'d_model': 2048, 'n_heads': 16, 'n_kv_heads': None, 'clip_qkv': None, 'n_layers': 16, 'mlp_ratio': 1, 'mlp_hidden_size': None, 'activation_type': 'swiglu', 'block_type': 'moe', 'block_group_size': 1, 'alibi': False, 'alibi_bias_max': 8.0, 'rope': True, 'rope_full_precision': True, 'rope_theta': 10000, 'flash_attention': True, 'attention_dropout': 0.0, 'multi_query_attention': None, 'attention_layer_norm': True, 'residual_dropout': 0.0, 'embedding_dropout': 0.0, 'embedding_layer_norm': False, 'layer_norm_type': 'rms', 'layer_norm_with_affine': True, 'layer_norm_eps': 1e-05, 'attention_layer_norm_with_affine': True, 'max_sequence_length': 4096, 'include_bias': False, 'bias_for_layer_norm': False, 'scale_logits': False, 'vocab_size': 50280, 'embedding_size': 50304, 'weight_tying': False, 'eos_token_id': 0, 'pad_token_id': 1, 'init_device': 'meta', 'init_fn': 'normal', 'init_std': 0.02, 'init_cutoff_factor': 3.0, 'precision': 'amp_bf16', 'moe_num_experts': 64, 'moe_top_k': 8, 'moe_mlp_impl': 'sparse', 'moe_log_expert_assignment': False, 'moe_shared_expert': False, 'moe_lbl_in_fp32': False, 'moe_loss_weight': 0.01, 'moe_zloss_weight': 0.001, 'moe_dropless': False, 'moe_capacity_factor': 1.25, 'himoe_enabled': False, 'himoe_n_groups': 4, 'himoe_top_k_per_group': 2, 'himoe_lambda1': 0.01, 'himoe_lambda2': 0.1, 'scale_emb_init': False, 'emb_init_std': None, 'norm_after': False}, 'optimizer': {'name': 'adamw', 'learning_rate': 0.0004, 'weight_decay': 0.1, 'betas': (0.9, 0.95), 'eps': 1e-08, 'no_decay_norm_and_bias': None, 'selective_updates': False, 'decay_norm_and_bias': True, 'decay_embeddings': True, 'metrics_log_interval': 10, 'record_update_metrics': False}, 'scheduler': {'name': 'cosine_with_warmup', 'units': 'tokens', 't_warmup': 1405296000, 't_max': 58540000000, 'alpha_f': 0.1, 'grad_clip_warmup_steps': None, 'grad_clip_warmup_factor': None, 'warmup_min_lr': None}, 'data': {'paths': ['/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-13-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-14-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-15-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-000-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-001-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-002-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-003-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-004-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-005-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-006-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-007-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-008-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-009-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-010-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-011-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-012-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-013-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy'], 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 8, 'drop_last': True, 'pin_memory': True, 'prefetch_factor': 8, 'persistent_workers': True, 'timeout': 0, 'seed': None, 'instance_filter': {'repetition_max_period': 13, 'repetition_min_period': 1, 'repetition_max_count': 32}}, 'restore_dataloader': True, 'fast_forward_batches': None, 'evaluators': [{'label': 'all-small-ppl-validation', 'type': 'lm', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': {'c4_en-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/c4_en/val/part-0-00000.npy'], 'dolma_books-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_books/val/part-0-00000.npy'], 'dolma_common-crawl-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_common-crawl/val/part-0-00000.npy'], 'dolma_pes2o-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_pes2o/val/part-0-00000.npy'], 'dolma_reddit-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_reddit/val/part-0-00000.npy'], 'dolma_stack-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_stack/val/part-0-00000.npy'], 'dolma_wiki-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_wiki/val/part-0-00000.npy'], 'ice-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/ice/val/part-0-00000.npy'], 'm2d2_s2orc-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/m2d2_s2orc/val/part-0-00000.npy'], 'pile-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/pile/val/part-0-00000.npy'], 'wikitext_103-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/wikitext_103/val/part-0-00000.npy']}, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': True, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'piqa', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'hellaswag', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'winogrande', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'openbook_qa', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'boolq', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'sciq', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'arc_easy', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'arc_challenge', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'copa', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'commonsense_qa', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'social_iqa', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_stem_var', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_humanities_var', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_social_sciences_var', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_other_var', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_stem_mc_5shot', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_humanities_mc_5shot', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_social_sciences_mc_5shot', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_other_mc_5shot', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_stem_mc_5shot_test', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_humanities_mc_5shot_test', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_social_sciences_mc_5shot_test', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_other_mc_5shot_test', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}], 'eval_interval': 2000, 'tokenizer': {'identifier': 'tokenizers/allenai_gpt-neox-olmo-dolma-v1_5.json', 'truncate_direction': 'right'}, 'save_folder': 'runs/olmoe-7b-29b-2ep', 'remote_save_folder': None, 'canceled_check_interval': 50, 'save_interval': 2000, 'save_interval_unsharded': None, 'save_interval_ephemeral': None, 'save_num_checkpoints_to_keep': 5, 'save_num_unsharded_checkpoints_to_keep': -1, 'save_overwrite': True, 'force_save_unsharded': False, 'no_pre_train_checkpoint': True, 'load_path': None, 'load_path_sharded_checkpointer': None, 'try_load_latest_save': False, 'reset_optimizer_state': False, 'reset_trainer_state': False, 'sharded_checkpointer': 'olmo_core', 'new_style_checkpoints': None, 'max_duration': '2ep', 'global_train_batch_size': 1024, 'device_train_batch_size': 256, 'device_train_microbatch_size': 2, 'device_eval_batch_size': 1, 'eval_subset_num_batches': -1, 'eval_on_load': False, 'device_train_grad_accum': 128, 'max_grad_norm': 1.0, 'max_grad_norm_ratio': None, 'precision': 'amp_bf16', 'speed_monitor': {'window_size': 1, 'gpu_flops_available': None}, 'console_log_interval': 1, 'gen1_gc_interval': 1, 'compile': None, 'distributed_strategy': 'fsdp', 'fsdp': {'use_orig_params': True, 'sharding_strategy': , 'wrapping_strategy': 'by_block', 'precision': 'mixed', 'hybrid_sharding_num_model_replicas': None}, 'ddp': None, 'softmax_auxiliary_loss': False, 'auxiliary_loss_multiplier': 0.0001, 'time_limit': None, 'extra_steps_after_cancel': 10, 'early_stopping_factor': None, 'save_data_indices': True, 'python_profiling': False, 'torch_profiling': False, 'stop_at': None, 'stop_after': None, 'activation_checkpointing': 'fine_grained', 'fused_loss': None, 'hf_datasets_cache_dir': None, 'module_outputs_save_steps': None, '_wandb': {'code_path': 'code/OLMo/scripts/train.py'}} +2025-12-19 03:28:09,457 INFO MainThread:1675715 [wandb_init.py:init():889] starting backend +2025-12-19 03:28:09,710 INFO MainThread:1675715 [wandb_init.py:init():892] sending inform_init request +2025-12-19 03:28:09,713 INFO MainThread:1675715 [wandb_init.py:init():900] backend started and connected +2025-12-19 03:28:09,716 INFO MainThread:1675715 [wandb_init.py:init():970] updated telemetry +2025-12-19 03:28:09,718 INFO MainThread:1675715 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout +2025-12-19 03:28:10,068 INFO MainThread:1675715 [wandb_init.py:init():1041] starting run threads in backend +2025-12-19 03:28:10,155 INFO MainThread:1675715 [wandb_run.py:_console_start():2521] atexit reg +2025-12-19 03:28:10,155 INFO MainThread:1675715 [wandb_run.py:_redirect():2369] redirect: wrap_raw +2025-12-19 03:28:10,155 INFO MainThread:1675715 [wandb_run.py:_redirect():2438] Wrapping output streams. +2025-12-19 03:28:10,155 INFO MainThread:1675715 [wandb_run.py:_redirect():2461] Redirects installed. +2025-12-19 03:28:10,158 INFO MainThread:1675715 [wandb_init.py:init():1081] run started, returning control to user process +2025-12-25 13:58:11,885 INFO MainThread:1675715 [wandb_run.py:_finish():2287] finishing run bulgakov/hiomoe/fhag09gx +2025-12-25 13:58:11,885 INFO MainThread:1675715 [wandb_run.py:_atexit_cleanup():2486] got exitcode: 0 +2025-12-25 13:58:11,885 INFO MainThread:1675715 [wandb_run.py:_restore():2468] restore +2025-12-25 13:58:11,885 INFO MainThread:1675715 [wandb_run.py:_restore():2474] restore done +2025-12-25 13:58:15,403 INFO MainThread:1675715 [wandb_run.py:_footer_sync_info():3862] logging synced files diff --git a/wandb/wandb/run-20251219_032809-fhag09gx/files/code/OLMo/scripts/train.py b/wandb/wandb/run-20251219_032809-fhag09gx/files/code/OLMo/scripts/train.py new file mode 100644 index 0000000000000000000000000000000000000000..eeb6a82e4728c477b9f5772e83543534a18e4f42 --- /dev/null +++ b/wandb/wandb/run-20251219_032809-fhag09gx/files/code/OLMo/scripts/train.py @@ -0,0 +1,373 @@ +"""Run this script with 'torchrun'.""" + +import gzip +import logging +import sys +from datetime import timedelta +from pathlib import Path +from typing import Optional, TextIO + +import torch +import torch.distributed as dist +import torch.multiprocessing as mp +import wandb +from packaging import version +from torch.distributed.fsdp import FullyShardedDataParallel as FSDP +from torch.distributed.fsdp import ShardingStrategy +from torch.nn.parallel import DistributedDataParallel as DDP + +from olmo.config import ( + ActivationCheckpointingStrategy, + CheckpointType, + DDPGradSyncMode, + DistributedStrategy, + TrainConfig, +) +from olmo.data import build_train_dataloader +from olmo.eval import build_evaluators +from olmo.exceptions import OLMoCliError, OLMoConfigurationError +from olmo.model import OLMo +from olmo.optim import BoltOnWarmupScheduler, build_optimizer, build_scheduler +from olmo.torch_util import ( + barrier, + get_default_device, + get_global_rank, + get_local_rank, + get_local_world_size, + get_world_size, + peak_gpu_memory, + seed_all, +) +from olmo.train import Trainer +from olmo.util import ( + add_cached_path_clients, + clean_opt, + find_latest_checkpoint, + log_extra_field, + prepare_cli_environment, +) + +log = logging.getLogger("train") + + +def main(cfg: TrainConfig) -> None: + # Ensure run name set. + if cfg.run_name is None: + raise OLMoConfigurationError("--run_name is required") + log_extra_field("run_name", cfg.run_name) + + # Sanity check + if (cfg.reset_optimizer_state or cfg.reset_trainer_state) and cfg.load_path is None: + log.warning( + "You want to reset the optimizer or trainer state, but we're not loading from the checkpoint. The" + "setting has no effect." + ) + + barrier() + + device = torch.device("cuda") + + # Fill some configuration options. + cfg.model.precision = cfg.precision + cfg.device_train_batch_size = cfg.global_train_batch_size // get_world_size() + assert cfg.device_train_batch_size is not None # for mypy + cfg.device_train_grad_accum = cfg.device_train_batch_size // cfg.device_train_microbatch_size + if cfg.optimizer.no_decay_norm_and_bias is not None: + log.warning( + "You set the deprecated config option `no_decay_norm_and_bias`. For compatibility, this" + "setting will take precedence over all other weight decay configurations. Please change" + "your config to use `decay_norm_and_bias` and `decay_embeddings` instead." + ) + cfg.optimizer.decay_norm_and_bias = not cfg.optimizer.no_decay_norm_and_bias + cfg.optimizer.decay_embeddings = not cfg.optimizer.no_decay_norm_and_bias + cfg.optimizer.no_decay_norm_and_bias = None # So nobody uses this by accident. + + # Display and save configuration. + if get_global_rank() == 0: + if cfg.data.paths is not None and len(cfg.data.paths) < 50: + log.info("Configuration:") + log.info(cfg) + if not cfg.dry_run and (cfg.load_path is None or Path(cfg.load_path).parent != Path(cfg.save_folder)): + # Save config. + save_path = Path(cfg.save_folder) / "config.yaml" + if save_path.is_file() and not cfg.save_overwrite: + raise OLMoConfigurationError(f"{save_path} already exists, use --save_overwrite to overwrite") + else: + log.info(f"Saving config to {save_path}") + save_path.parent.mkdir(exist_ok=True, parents=True) + cfg.save(save_path) + del save_path + + barrier() + + # Maybe start W&B run. + if cfg.wandb is not None and (get_global_rank() == 0 or not cfg.wandb.rank_zero_only): + wandb_dir = Path(cfg.save_folder) / "wandb" + wandb_dir.mkdir(parents=True, exist_ok=True) + wandb.init( + dir=wandb_dir, + project=cfg.wandb.project, + entity=cfg.wandb.entity, + group=cfg.wandb.group, + name=cfg.wandb.name, + tags=cfg.wandb.tags, + config=cfg.asdict(exclude=["wandb"]), + ) + + barrier() + + # Set seed. + seed_all(cfg.seed) + + # Construct data loader. + train_loader = build_train_dataloader(cfg) + + # Construct evaluators. + evaluators = build_evaluators(cfg, device) + barrier() + + # Initialize the model. + log.info("Building model...") + if ( + cfg.model.block_type == "moe" + and cfg.activation_checkpointing + and cfg.activation_checkpointing != ActivationCheckpointingStrategy.fine_grained + ): + raise OLMoConfigurationError( + "Only no or fine-grained activation checkpointing is supported for MoE models." + ) + + olmo_model = OLMo(cfg.model) + log.info(f"Total number of parameters: {olmo_model.num_params():,d}") + log.info(f"Number of non-embedding parameters: {olmo_model.num_params(include_embedding=False):,d}") + if olmo_model.config.block_type == "moe": + log.info(f"Number of active parameters: {olmo_model.num_params(include_inactive_params=False):,d}") + log.info(f"Peak GPU Memory (MB) before {cfg.distributed_strategy}: {int(peak_gpu_memory() or 0)}") + + olmo_model.set_activation_checkpointing(cfg.activation_checkpointing) + + if cfg.distributed_strategy == DistributedStrategy.ddp: + log.info("Wrapping model with DDP...") + assert cfg.ddp is not None, "DistributedStrategy ddp needs cfg.ddp to be set!" + + if cfg.model.init_device != "cuda": + raise OLMoConfigurationError("DDP does not work with init_device set to anything other than `cuda`.") + + if cfg.ddp.find_unused_params is True and cfg.ddp.grad_sync_mode != DDPGradSyncMode.micro_batch: + raise OLMoConfigurationError( + "`find_unused_params` is set to True. DDP needs to synchronize gradients for every micro-batch to avoid errors. Set `grad_sync_mode` to `micro_batch`." + ) + + param_init_fn = None + + # move to cuda before calling ddp + dist_model = DDP(olmo_model.to(device), find_unused_parameters=cfg.ddp.find_unused_params) + elif cfg.distributed_strategy == DistributedStrategy.fsdp: + # Wrap the model in FSDP. + log.info("Wrapping model with FSDP...") + assert cfg.fsdp is not None, "DistributedStrategy fsdp needs cfg.fsdp to be set!" + wrap_policy = olmo_model.get_fsdp_wrap_policy(cfg.fsdp.wrapping_strategy) + + if version.parse(torch.__version__) >= version.parse("2.1.0"): + # This prevents any parameters from being initialized twice + def dummy_init_fn(module: torch.nn.Module) -> None: + module.to_empty(device=get_default_device()) + + param_init_fn = dummy_init_fn + else: + param_init_fn = None + + # Set up device mesh for hybrid sharding in order to specify which nodes are assoicated to a given model replica + device_mesh = None + hybrid_sharding_fsdp_kwargs = {} + if cfg.fsdp.sharding_strategy in (ShardingStrategy.HYBRID_SHARD, ShardingStrategy._HYBRID_SHARD_ZERO2): + if version.parse(torch.__version__) < version.parse("2.2.0"): + # Device mesh was not added to PyTorch until v2.2.0 + raise OLMoConfigurationError( + "OLMo training does not correctly support hybrid sharding before torch 2.2.0" + ) + + from torch.distributed.device_mesh import init_device_mesh + + num_model_replicas = cfg.fsdp.hybrid_sharding_num_model_replicas or ( + get_world_size() // get_local_world_size() + ) + + if num_model_replicas <= 0: + raise OLMoConfigurationError("fsdp.hybrid_sharding_num_model_replicas must be a positive integer") + + if get_world_size() % num_model_replicas != 0: + raise OLMoConfigurationError("fsdp.hybrid_sharding_num_model_replicas must divide world size") + + device_mesh = init_device_mesh("cuda", (num_model_replicas, get_world_size() // num_model_replicas)) + hybrid_sharding_fsdp_kwargs["device_mesh"] = device_mesh + + dist_model = FSDP( + olmo_model, + sharding_strategy=cfg.fsdp.sharding_strategy, + mixed_precision=cfg.fsdp_precision, + auto_wrap_policy=wrap_policy, + use_orig_params=cfg.fsdp.use_orig_params, # needed for compile and some of our optimizer/parameter metrics + limit_all_gathers=True, + device_id=get_local_rank(), + param_init_fn=param_init_fn, + **hybrid_sharding_fsdp_kwargs, + ) + elif cfg.distributed_strategy is None: + raise NotImplementedError("Single accelerator training not implemented yet!") + + # when param_init_fn is None, FSDP will call reset_parameters() automatically + if param_init_fn is not None or cfg.distributed_strategy == DistributedStrategy.ddp: + olmo_model.reset_parameters() + + log.info(f"Peak GPU Memory (MB) after {cfg.distributed_strategy}: {int(peak_gpu_memory() or 0)}") + log.info("Model:") + log.info(dist_model) + + # Construct optimizer and learning rate scheduler. + optim = build_optimizer(cfg, dist_model) + scheduler = build_scheduler(cfg) + + # Data indices file. + indices_file: Optional[TextIO] = None + if cfg.save_data_indices: + indices_file_path = Path(cfg.save_folder) / f"data-indices/rank{get_global_rank()}.tsv.gz" + if indices_file_path.exists() and not cfg.save_overwrite: + raise OLMoConfigurationError(f"{indices_file_path} already exists, use --save_overwrite to overwrite") + indices_file_path.parent.mkdir(exist_ok=True, parents=True) + indices_file = gzip.open(indices_file_path, "wt") + + # Consolidate components into `Trainer` object. + with Trainer( + cfg=cfg, + epoch=cfg.epoch, + model=olmo_model, + dist_model=dist_model, + optim=optim, + scheduler=scheduler, + train_loader=train_loader, + device=device, + evaluators=evaluators, + indices_file=indices_file, + ) as trainer: + if cfg.try_load_latest_save: + if ( + cfg.save_folder is not None + and (checkpoint_dir := find_latest_checkpoint(cfg.save_folder)) is not None + ): + log.info("Setting load path to local checkpoint %s", checkpoint_dir) + cfg.load_path = str(checkpoint_dir) + elif ( + cfg.remote_save_folder is not None + and (checkpoint_dir := find_latest_checkpoint(cfg.remote_save_folder)) is not None + ): + log.info("Setting load path to remote checkpoint %s", checkpoint_dir) + cfg.load_path = str(checkpoint_dir) + + if not cfg.dry_run and not cfg.no_pre_train_checkpoint and cfg.load_path is None: + if cfg.distributed_strategy == DistributedStrategy.ddp: + checkpoint_type = CheckpointType.unsharded + + if cfg.save_interval_unsharded is None: + log.warning( + "DDP requires setting `save_interval_unsharded`. Using the value set for `save_interval`." + ) + cfg.save_interval_unsharded = cfg.save_interval + + if cfg.save_num_unsharded_checkpoints_to_keep == 0: + log.warning( + "DDP requires setting `save_num_unsharded_checkpoints_to_keep`. Using the value set for `save_num_checkpoints_to_keep`." + ) + cfg.save_num_unsharded_checkpoints_to_keep = cfg.save_num_checkpoints_to_keep + elif cfg.distributed_strategy == DistributedStrategy.fsdp: + checkpoint_type = ( + CheckpointType.sharded if cfg.save_num_checkpoints_to_keep != 0 else CheckpointType.unsharded + ) + else: + raise NotImplementedError(f"Distributed strategy {cfg.distributed_strategy} not supported yet!") + + # We save a checkpoint up-front to make sure this won't fail (due to disk space or whatever). + log.info("Saving pre-train checkpoint...") + checkpoint_path, local_checkpoint_cache = trainer.save_checkpoint(checkpoint_type=checkpoint_type) + log.info(f"Checkpoint saved to {checkpoint_path}") + + # And they we verify that we can load it. + log.info("Attempting to load pre-train checkpoint...") + trainer.restore_checkpoint( + checkpoint_path, checkpoint_type=checkpoint_type, local_cache=local_checkpoint_cache + ) + log.info("Checkpoint successfully loaded") + + # NOTE: https://github.com/allenai/LLM/issues/233 + # log.info("Removing pre-train checkpoint...") + # trainer.remove_checkpoint(checkpoint_type=checkpoint_type) + # log.info("Successfully removed checkpoint") + + if cfg.load_path is not None: + log.info(f"Loading checkpoint from {cfg.load_path}...") + trainer.restore_checkpoint( + cfg.load_path, + load_optimizer_state=not cfg.reset_optimizer_state, + load_trainer_state=not cfg.reset_trainer_state, + sharded_checkpointer=cfg.load_path_sharded_checkpointer, + ) + log.info("Checkpoint successfully loaded") + + # If we have to, set a new scheduler: + if cfg.reset_optimizer_state and not cfg.reset_trainer_state: + trainer.scheduler = BoltOnWarmupScheduler.wrap( + trainer.scheduler, + trainer.global_step, + int(trainer.global_step + cfg.scheduler.t_warmup), + ) + + if cfg.force_save_unsharded and cfg.distributed_strategy != DistributedStrategy.ddp: + log.info("Saving unsharded checkpoint...") + checkpoint_path, _ = trainer.save_checkpoint(checkpoint_type=CheckpointType.unsharded) + log.info(f"Unsharded checkpoint saved to {checkpoint_path}") + + if cfg.compile is not None: + # TODO (epwalsh): trying to compile the whole train step results in a compile-time error from within + # the optimizer. We should investigate this further at some point. + # trainer.train_step = torch.compile(trainer.train_step, **cfg.compile.asdict()) + trainer.train_batch = torch.compile(trainer.train_batch, **cfg.compile.asdict()) # type: ignore + # TODO (epwalsh): compiling the `eval_batch()` method is a little sketchy since the inputs will look + # different for different eval tasks. That might be okay, but it might not be. + # trainer.eval_batch = torch.compile(trainer.eval_batch, **cfg.compile.asdict()) # type: ignore + # Alternatively, could just do this: + # trainer.fsdp_model = torch.compile(trainer.fsdp_model, **cfg.compile.asdict()) + + if not cfg.dry_run: + log.info("Starting training...") + trainer.fit() + log.info("Training complete") + else: + log.info("Dry run complete") + + +if __name__ == "__main__": + try: + mp.set_start_method("spawn", force=True) + except RuntimeError as e: + print(f"failed to set multiprocessing start method: {e}") + log.info(f"Multiprocessing start method set to '{mp.get_start_method()}'") + + # Set CUDA device. + torch.cuda.set_device(f"cuda:{get_local_rank()}") + + # Initialize process group. + dist.init_process_group(backend="nccl", timeout=timedelta(minutes=30)) + log.info("Process group initialized") + + prepare_cli_environment() + log.info("CLI environment prepared") + + add_cached_path_clients() + + try: + yaml_path, args_list = sys.argv[1], sys.argv[2:] + except IndexError: + raise OLMoCliError(f"Usage: {sys.argv[0]} [CONFIG_PATH] [OPTIONS]") + + cfg = TrainConfig.load(yaml_path, [clean_opt(s) for s in args_list]) + main(cfg) diff --git a/wandb/wandb/run-20251219_032809-fhag09gx/files/config.yaml b/wandb/wandb/run-20251219_032809-fhag09gx/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a74fb189fe49e92b7e30c24104e3459ba5b615d --- /dev/null +++ b/wandb/wandb/run-20251219_032809-fhag09gx/files/config.yaml @@ -0,0 +1,883 @@ +_wandb: + value: + cli_version: 0.23.1 + code_path: code/OLMo/scripts/train.py + e: + hkx9xgphpmzezm5xyyncldepfa9kpirs: + args: + - configs/test/OLMoE-7B-500GB-test.yaml + - --run_name=olmoe-7b-29b-2ep + - --save_overwrite + codePath: OLMo/scripts/train.py + codePathLocal: scripts/train.py + cpu_count: 96 + cpu_count_logical: 192 + cudaVersion: "13.0" + disk: + /: + total: "422146228224" + used: "65563865088" + email: alexmyasnikov451@gmail.com + executable: /data/users/xandi281/work/Hi-OLMoE/venv/bin/python3 + git: + commit: 9b3ad56c62f3b7b0e435efc757c49704db6e6333 + remote: git@github.com:AverageMetaheuristicsEnjoyer/Hi-OLMoE.git + gpu: NVIDIA H200 + gpu_count: 8 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-28548a6f-a458-cff0-d4d9-cd8992f7bfc8 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-390840f6-de3c-1ecc-1353-cf4d01ab89e2 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-bbba09e7-ef87-452c-7d34-cb2a03510f2c + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-722903bc-18d9-e3ef-0146-caff9c35e5bf + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-f09af96c-4a99-e9b3-2e01-4d03d1a959dd + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-0aa55090-5ce5-56e2-62cc-36bc2ae4fb77 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-8a81a780-cddb-5b47-490f-871873b601ec + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-81dfcc32-ea97-c0f1-778e-ff44c4992c1a + host: ubuntu-gpu-pod-final + memory: + total: "2163971334144" + os: Linux-5.15.0-160-generic-x86_64-with-glibc2.39 + program: /data/users/xandi281/work/Hi-OLMoE/OLMo/scripts/train.py + python: CPython 3.11.14 + root: runs/olmoe-7b-29b-2ep/wandb + startedAt: "2025-12-19T03:28:09.455870Z" + writerId: hkx9xgphpmzezm5xyyncldepfa9kpirs + m: [] + python_version: 3.11.14 + t: + "1": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + "2": + - 1 + - 5 + - 11 + - 49 + - 51 + - 53 + "3": + - 2 + - 13 + - 15 + - 16 + - 61 + "4": 3.11.14 + "5": 0.23.1 + "6": 4.57.3 + "10": + - 19 + "12": 0.23.1 + "13": linux-x86_64 +activation_checkpointing: + value: fine_grained +auxiliary_loss_multiplier: + value: 0.0001 +canceled_check_interval: + value: 50 +compile: + value: null +console_log_interval: + value: 1 +data: + value: + datasets: null + drop_last: true + generate_attention_mask: false + generate_doc_lengths: false + instance_filter: + repetition_max_count: 32 + repetition_max_period: 13 + repetition_min_period: 1 + label_mask_paths: null + memmap_dtype: uint16 + num_workers: 8 + pad_direction: right + paths: + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-13-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-14-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-15-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-000-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-001-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-002-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-003-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-004-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-005-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-006-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-007-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-008-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-009-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-010-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-011-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-012-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-013-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy + - /data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy + persistent_workers: true + pin_memory: true + prefetch_factor: 8 + seed: null + timeout: 0 +ddp: + value: null +device_eval_batch_size: + value: 1 +device_train_batch_size: + value: 256 +device_train_grad_accum: + value: 128 +device_train_microbatch_size: + value: 2 +distributed_strategy: + value: fsdp +dry_run: + value: false +early_stopping_factor: + value: null +epoch: + value: null +eval_interval: + value: 2000 +eval_on_load: + value: false +eval_subset_num_batches: + value: -1 +evaluators: + value: + - data: + datasets: + c4_en-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/c4_en/val/part-0-00000.npy + dolma_books-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_books/val/part-0-00000.npy + dolma_common-crawl-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_common-crawl/val/part-0-00000.npy + dolma_pes2o-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_pes2o/val/part-0-00000.npy + dolma_reddit-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_reddit/val/part-0-00000.npy + dolma_stack-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_stack/val/part-0-00000.npy + dolma_wiki-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_wiki/val/part-0-00000.npy + ice-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/ice/val/part-0-00000.npy + m2d2_s2orc-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/m2d2_s2orc/val/part-0-00000.npy + pile-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/pile/val/part-0-00000.npy + wikitext_103-validation: + - /data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/wikitext_103/val/part-0-00000.npy + drop_last: true + generate_attention_mask: false + generate_doc_lengths: false + instance_filter: null + label_mask_paths: null + memmap_dtype: uint16 + num_workers: 0 + pad_direction: right + paths: null + persistent_workers: false + pin_memory: false + prefetch_factor: null + seed: null + timeout: 0 + device_eval_batch_size: null + label: all-small-ppl-validation + subset_num_batches: null + type: lm + - data: + datasets: null + drop_last: false + generate_attention_mask: false + generate_doc_lengths: false + instance_filter: null + label_mask_paths: null + memmap_dtype: uint16 + num_workers: 0 + pad_direction: right + paths: null + persistent_workers: false + pin_memory: false + prefetch_factor: null + seed: null + timeout: 0 + device_eval_batch_size: null + label: piqa + subset_num_batches: null + type: downstream + - data: + datasets: null + drop_last: false + generate_attention_mask: false + generate_doc_lengths: false + instance_filter: null + label_mask_paths: null + memmap_dtype: uint16 + num_workers: 0 + pad_direction: right + paths: null + persistent_workers: false + pin_memory: false + prefetch_factor: null + seed: null + timeout: 0 + device_eval_batch_size: null + label: hellaswag + subset_num_batches: null + type: downstream + - data: + datasets: null + drop_last: false + generate_attention_mask: false + generate_doc_lengths: false + instance_filter: null + label_mask_paths: null + memmap_dtype: uint16 + num_workers: 0 + pad_direction: right + paths: null + persistent_workers: false + pin_memory: false + prefetch_factor: null + seed: null + timeout: 0 + device_eval_batch_size: null + label: winogrande + subset_num_batches: null + type: downstream + - data: + datasets: null + drop_last: false + generate_attention_mask: false + generate_doc_lengths: false + instance_filter: null + label_mask_paths: null + memmap_dtype: uint16 + num_workers: 0 + pad_direction: right + paths: null + persistent_workers: false + pin_memory: false + prefetch_factor: null + seed: null + timeout: 0 + device_eval_batch_size: null + label: openbook_qa + subset_num_batches: null + type: downstream + - data: + datasets: null + drop_last: false + generate_attention_mask: false + generate_doc_lengths: false + instance_filter: null + label_mask_paths: null + memmap_dtype: uint16 + num_workers: 0 + pad_direction: right + paths: null + persistent_workers: false + pin_memory: false + prefetch_factor: null + seed: null + timeout: 0 + device_eval_batch_size: null + label: boolq + subset_num_batches: null + type: downstream + - data: + datasets: null + drop_last: false + generate_attention_mask: false + generate_doc_lengths: false + instance_filter: null + label_mask_paths: null + memmap_dtype: uint16 + num_workers: 0 + pad_direction: right + paths: null + persistent_workers: false + pin_memory: false + prefetch_factor: null + seed: null + timeout: 0 + device_eval_batch_size: null + label: sciq + subset_num_batches: null + type: downstream + - data: + datasets: null + drop_last: false + generate_attention_mask: false + generate_doc_lengths: false + instance_filter: null + label_mask_paths: null + memmap_dtype: uint16 + num_workers: 0 + pad_direction: right + paths: null + persistent_workers: false + pin_memory: false + prefetch_factor: null + seed: null + timeout: 0 + device_eval_batch_size: null + label: arc_easy + subset_num_batches: null + type: downstream + - data: + datasets: null + drop_last: false + generate_attention_mask: false + generate_doc_lengths: false + instance_filter: null + label_mask_paths: null + memmap_dtype: uint16 + num_workers: 0 + pad_direction: right + paths: null + persistent_workers: false + pin_memory: false + prefetch_factor: null + seed: null + timeout: 0 + device_eval_batch_size: null + label: arc_challenge + subset_num_batches: null + type: downstream + - data: + datasets: null + drop_last: false + generate_attention_mask: false + generate_doc_lengths: false + instance_filter: null + label_mask_paths: null + memmap_dtype: uint16 + num_workers: 0 + pad_direction: right + paths: null + persistent_workers: false + pin_memory: false + prefetch_factor: null + seed: null + timeout: 0 + device_eval_batch_size: null + label: copa + subset_num_batches: null + type: downstream + - data: + datasets: null + drop_last: false + generate_attention_mask: false + generate_doc_lengths: false + instance_filter: null + label_mask_paths: null + memmap_dtype: uint16 + num_workers: 0 + pad_direction: right + paths: null + persistent_workers: false + pin_memory: false + prefetch_factor: null + seed: null + timeout: 0 + device_eval_batch_size: null + label: commonsense_qa + subset_num_batches: null + type: downstream + - data: + datasets: null + drop_last: false + generate_attention_mask: false + generate_doc_lengths: false + instance_filter: null + label_mask_paths: null + memmap_dtype: uint16 + num_workers: 0 + pad_direction: right + paths: null + persistent_workers: false + pin_memory: false + prefetch_factor: null + seed: null + timeout: 0 + device_eval_batch_size: null + label: social_iqa + subset_num_batches: null + type: downstream + - data: + datasets: null + drop_last: false + generate_attention_mask: false + generate_doc_lengths: false + instance_filter: null + label_mask_paths: null + memmap_dtype: uint16 + num_workers: 0 + pad_direction: right + paths: null + persistent_workers: false + pin_memory: false + prefetch_factor: null + seed: null + timeout: 0 + device_eval_batch_size: null + label: mmlu_stem_var + subset_num_batches: null + type: downstream + - data: + datasets: null + drop_last: false + generate_attention_mask: false + generate_doc_lengths: false + instance_filter: null + label_mask_paths: null + memmap_dtype: uint16 + num_workers: 0 + pad_direction: right + paths: null + persistent_workers: false + pin_memory: false + prefetch_factor: null + seed: null + timeout: 0 + device_eval_batch_size: null + label: mmlu_humanities_var + subset_num_batches: null + type: downstream + - data: + datasets: null + drop_last: false + generate_attention_mask: false + generate_doc_lengths: false + instance_filter: null + label_mask_paths: null + memmap_dtype: uint16 + num_workers: 0 + pad_direction: right + paths: null + persistent_workers: false + pin_memory: false + prefetch_factor: null + seed: null + timeout: 0 + device_eval_batch_size: null + label: mmlu_social_sciences_var + subset_num_batches: null + type: downstream + - data: + datasets: null + drop_last: false + generate_attention_mask: false + generate_doc_lengths: false + instance_filter: null + label_mask_paths: null + memmap_dtype: uint16 + num_workers: 0 + pad_direction: right + paths: null + persistent_workers: false + pin_memory: false + prefetch_factor: null + seed: null + timeout: 0 + device_eval_batch_size: null + label: mmlu_other_var + subset_num_batches: null + type: downstream + - data: + datasets: null + drop_last: false + generate_attention_mask: false + generate_doc_lengths: false + instance_filter: null + label_mask_paths: null + memmap_dtype: uint16 + num_workers: 0 + pad_direction: right + paths: null + persistent_workers: false + pin_memory: false + prefetch_factor: null + seed: null + timeout: 0 + device_eval_batch_size: null + label: mmlu_stem_mc_5shot + subset_num_batches: null + type: downstream + - data: + datasets: null + drop_last: false + generate_attention_mask: false + generate_doc_lengths: false + instance_filter: null + label_mask_paths: null + memmap_dtype: uint16 + num_workers: 0 + pad_direction: right + paths: null + persistent_workers: false + pin_memory: false + prefetch_factor: null + seed: null + timeout: 0 + device_eval_batch_size: null + label: mmlu_humanities_mc_5shot + subset_num_batches: null + type: downstream + - data: + datasets: null + drop_last: false + generate_attention_mask: false + generate_doc_lengths: false + instance_filter: null + label_mask_paths: null + memmap_dtype: uint16 + num_workers: 0 + pad_direction: right + paths: null + persistent_workers: false + pin_memory: false + prefetch_factor: null + seed: null + timeout: 0 + device_eval_batch_size: null + label: mmlu_social_sciences_mc_5shot + subset_num_batches: null + type: downstream + - data: + datasets: null + drop_last: false + generate_attention_mask: false + generate_doc_lengths: false + instance_filter: null + label_mask_paths: null + memmap_dtype: uint16 + num_workers: 0 + pad_direction: right + paths: null + persistent_workers: false + pin_memory: false + prefetch_factor: null + seed: null + timeout: 0 + device_eval_batch_size: null + label: mmlu_other_mc_5shot + subset_num_batches: null + type: downstream + - data: + datasets: null + drop_last: false + generate_attention_mask: false + generate_doc_lengths: false + instance_filter: null + label_mask_paths: null + memmap_dtype: uint16 + num_workers: 0 + pad_direction: right + paths: null + persistent_workers: false + pin_memory: false + prefetch_factor: null + seed: null + timeout: 0 + device_eval_batch_size: null + label: mmlu_stem_mc_5shot_test + subset_num_batches: null + type: downstream + - data: + datasets: null + drop_last: false + generate_attention_mask: false + generate_doc_lengths: false + instance_filter: null + label_mask_paths: null + memmap_dtype: uint16 + num_workers: 0 + pad_direction: right + paths: null + persistent_workers: false + pin_memory: false + prefetch_factor: null + seed: null + timeout: 0 + device_eval_batch_size: null + label: mmlu_humanities_mc_5shot_test + subset_num_batches: null + type: downstream + - data: + datasets: null + drop_last: false + generate_attention_mask: false + generate_doc_lengths: false + instance_filter: null + label_mask_paths: null + memmap_dtype: uint16 + num_workers: 0 + pad_direction: right + paths: null + persistent_workers: false + pin_memory: false + prefetch_factor: null + seed: null + timeout: 0 + device_eval_batch_size: null + label: mmlu_social_sciences_mc_5shot_test + subset_num_batches: null + type: downstream + - data: + datasets: null + drop_last: false + generate_attention_mask: false + generate_doc_lengths: false + instance_filter: null + label_mask_paths: null + memmap_dtype: uint16 + num_workers: 0 + pad_direction: right + paths: null + persistent_workers: false + pin_memory: false + prefetch_factor: null + seed: null + timeout: 0 + device_eval_batch_size: null + label: mmlu_other_mc_5shot_test + subset_num_batches: null + type: downstream +extra_steps_after_cancel: + value: 10 +fast_forward_batches: + value: null +force_save_unsharded: + value: false +fsdp: + value: + hybrid_sharding_num_model_replicas: null + precision: mixed + sharding_strategy: FULL_SHARD + use_orig_params: true + wrapping_strategy: by_block +fused_loss: + value: null +gen1_gc_interval: + value: 1 +global_train_batch_size: + value: 1024 +hf_datasets_cache_dir: + value: null +load_path: + value: null +load_path_sharded_checkpointer: + value: null +max_duration: + value: 2ep +max_grad_norm: + value: 1 +max_grad_norm_ratio: + value: null +model: + value: + activation_type: swiglu + alibi: false + alibi_bias_max: 8 + attention_dropout: 0 + attention_layer_norm: true + attention_layer_norm_with_affine: true + bias_for_layer_norm: false + block_group_size: 1 + block_type: moe + clip_qkv: null + d_model: 2048 + emb_init_std: null + embedding_dropout: 0 + embedding_layer_norm: false + embedding_size: 50304 + eos_token_id: 0 + flash_attention: true + himoe_enabled: false + himoe_lambda1: 0.01 + himoe_lambda2: 0.1 + himoe_n_groups: 4 + himoe_top_k_per_group: 2 + include_bias: false + init_cutoff_factor: 3 + init_device: meta + init_fn: normal + init_std: 0.02 + layer_norm_eps: 1e-05 + layer_norm_type: rms + layer_norm_with_affine: true + max_sequence_length: 4096 + mlp_hidden_size: null + mlp_ratio: 1 + moe_capacity_factor: 1.25 + moe_dropless: false + moe_lbl_in_fp32: false + moe_log_expert_assignment: false + moe_loss_weight: 0.01 + moe_mlp_impl: sparse + moe_num_experts: 64 + moe_shared_expert: false + moe_top_k: 8 + moe_zloss_weight: 0.001 + multi_query_attention: null + n_heads: 16 + n_kv_heads: null + n_layers: 16 + norm_after: false + pad_token_id: 1 + precision: amp_bf16 + residual_dropout: 0 + rope: true + rope_full_precision: true + rope_theta: 10000 + scale_emb_init: false + scale_logits: false + vocab_size: 50280 + weight_tying: false +module_outputs_save_steps: + value: null +new_style_checkpoints: + value: null +no_pre_train_checkpoint: + value: true +optimizer: + value: + betas: + - 0.9 + - 0.95 + decay_embeddings: true + decay_norm_and_bias: true + eps: 1e-08 + learning_rate: 0.0004 + metrics_log_interval: 10 + name: adamw + no_decay_norm_and_bias: null + record_update_metrics: false + selective_updates: false + weight_decay: 0.1 +precision: + value: amp_bf16 +python_profiling: + value: false +remote_save_folder: + value: null +reset_optimizer_state: + value: false +reset_trainer_state: + value: false +restore_dataloader: + value: true +run_name: + value: olmoe-7b-29b-2ep +save_data_indices: + value: true +save_folder: + value: runs/olmoe-7b-29b-2ep +save_interval: + value: 2000 +save_interval_ephemeral: + value: null +save_interval_unsharded: + value: null +save_num_checkpoints_to_keep: + value: 5 +save_num_unsharded_checkpoints_to_keep: + value: -1 +save_overwrite: + value: true +scheduler: + value: + alpha_f: 0.1 + grad_clip_warmup_factor: null + grad_clip_warmup_steps: null + name: cosine_with_warmup + t_max: 58540000000 + t_warmup: 1405296000 + units: tokens + warmup_min_lr: null +seed: + value: 6198 +sharded_checkpointer: + value: olmo_core +softmax_auxiliary_loss: + value: false +speed_monitor: + value: + gpu_flops_available: null + window_size: 1 +stop_after: + value: null +stop_at: + value: null +time_limit: + value: null +tokenizer: + value: + identifier: tokenizers/allenai_gpt-neox-olmo-dolma-v1_5.json + truncate_direction: right +torch_profiling: + value: false +try_load_latest_save: + value: false diff --git a/wandb/wandb/run-20251219_032809-fhag09gx/files/output.log b/wandb/wandb/run-20251219_032809-fhag09gx/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..9b7a1e6d5000e45b2305f78feb48e47163812125 --- /dev/null +++ b/wandb/wandb/run-20251219_032809-fhag09gx/files/output.log @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0d408b475b86f6651a715c0f8d1688b1bcd94ecb011cfffa66ed89399020aa5 +size 36174013 diff --git a/wandb/wandb/run-20251219_032809-fhag09gx/files/requirements.txt b/wandb/wandb/run-20251219_032809-fhag09gx/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a19ee155c6b7003e8ef25c1b9d939a5455068822 --- /dev/null +++ b/wandb/wandb/run-20251219_032809-fhag09gx/files/requirements.txt @@ -0,0 +1,163 @@ +twine==6.2.0 +Pygments==2.19.2 +rsa==4.9.1 +black==23.12.1 +pyasn1_modules==0.4.2 +six==1.17.0 +nvidia-cufft==12.0.0.15 +pytest==9.0.2 +annotated-types==0.7.0 +id==1.5.0 +MarkupSafe==2.1.5 +wandb==0.23.1 +click-option-group==0.5.9 +petname==2.6 +ai2-olmo-core==0.1.0 +wheel==0.45.1 +pyarrow==22.0.0 +dataclass-extensions==0.2.4 +gitdb==4.0.12 +ftfy==6.3.1 +nvidia-nvshmem-cu13==3.3.24 +necessary==0.4.3 +ai2-olmo==0.5.0 +httpx==0.28.1 +lightning-utilities==0.15.2 +pytz==2025.2 +frozenlist==1.8.0 +hf-xet==1.2.0 +torchmetrics==1.8.2 +pytest-sphinx==0.6.3 +pydantic==2.12.5 +platformdirs==4.5.1 +urllib3==2.6.2 +iniconfig==2.3.0 +h11==0.16.0 +googleapis-common-protos==1.72.0 +nvidia-nvtx==13.0.39 +beaker-gantry==3.3.0 +nvidia-nvjitlink==13.0.39 +xxhash==3.6.0 +grpcio==1.76.0 +glom==24.11.0 +pycparser==2.23 +msgspec==0.20.0 +huggingface-hub==0.36.0 +protobuf==6.33.2 +mdurl==0.1.2 +triton==3.5.1 +python-dateutil==2.9.0.post0 +megablocks==0.5.1 +propcache==0.4.1 +nvidia-cuda-runtime==13.0.48 +readme_renderer==44.0 +fsspec==2025.10.0 +PyYAML==6.0.3 +face==24.0.0 +numpy==1.26.4 +more-itertools==10.8.0 +nvidia-cuda-nvrtc==13.0.48 +docutils==0.22.3 +tokenizers==0.22.1 +smmap==5.0.2 +jaraco.classes==3.4.0 +google-resumable-media==2.8.0 +trouting==0.3.3 +s3transfer==0.16.0 +keyring==25.7.0 +typing-inspection==0.4.2 +requirements-parser==0.13.0 +isort==5.12.0 +wrapt==2.0.1 +google-api-core==2.28.1 +beaker-py==2.5.3 +jeepney==0.9.0 +rfc3986==2.0.0 +tzdata==2025.2 +cachetools==6.2.2 +torch==2.9.1+cu130 +networkx==3.6.1 +google-crc32c==1.7.1 +click==8.3.1 +google-auth==2.43.0 +smart_open==7.5.0 +pydantic_core==2.41.5 +tomli==2.3.0 +markdown-it-py==4.0.0 +sentry-sdk==2.47.0 +tqdm==4.67.1 +wcwidth==0.2.14 +importlib_resources==6.5.2 +stanford-stk==0.7.0 +boltons==25.0.0 +GitPython==3.1.45 +nvidia-curand==10.4.0.35 +cryptography==46.0.3 +nvidia-cuda-cupti==13.0.48 +zipp==3.23.0 +boto3==1.42.8 +aiohappyeyeballs==2.6.1 +build==1.3.0 +proto-plus==1.26.1 +ruff==0.14.9 +filelock==3.20.0 +Jinja2==3.1.6 +pandas==2.3.3 +backports.tarfile==1.2.0 +google-cloud-core==2.5.0 +pyasn1==0.6.1 +nvidia-cusparse==12.6.2.49 +charset-normalizer==3.4.4 +transformers==4.57.3 +jaraco.context==6.0.1 +joblib==1.5.2 +regex==2025.11.3 +nvidia-nccl-cu13==2.27.7 +multidict==6.7.0 +jaraco.functools==4.3.0 +omegaconf==2.3.0 +mypy_extensions==1.1.0 +attrs==25.4.0 +nvidia-cublas==13.0.0.19 +nvidia-cufile==1.15.0.42 +pathspec==0.12.1 +botocore==1.42.8 +click-help-colors==0.9.4 +pluggy==1.6.0 +multiprocess==0.70.18 +idna==3.11 +aiosignal==1.4.0 +nh3==0.3.2 +nvidia-cusolver==12.0.3.29 +certifi==2025.11.12 +antlr4-python3-runtime==4.9.3 +httpcore==1.0.9 +google-cloud-storage==2.19.0 +threadpoolctl==3.6.0 +requests==2.32.5 +pyproject_hooks==1.2.0 +scikit-learn==1.8.0 +smashed==0.21.5 +requests-toolbelt==1.0.0 +importlib_metadata==8.7.0 +anyio==4.12.0 +cached_path==1.8.0 +mypy==1.3.0 +jmespath==1.0.1 +pip==24.0 +SecretStorage==3.5.0 +nvidia-cudnn-cu13==9.13.0.50 +aiohttp==3.13.2 +typing_extensions==4.15.0 +yarl==1.22.0 +mpmath==1.3.0 +packaging==24.1 +safetensors==0.7.0 +sympy==1.14.0 +scipy==1.16.3 +datasets==4.4.1 +dill==0.4.0 +nvidia-cusparselt-cu13==0.8.0 +setuptools==79.0.1 +rich==13.9.4 +cffi==2.0.0 diff --git a/wandb/wandb/run-20251219_032809-fhag09gx/files/wandb-metadata.json b/wandb/wandb/run-20251219_032809-fhag09gx/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..91f404df29807c96fb5fe6c5ea31a9894f5ee539 --- /dev/null +++ b/wandb/wandb/run-20251219_032809-fhag09gx/files/wandb-metadata.json @@ -0,0 +1,94 @@ +{ + "os": "Linux-5.15.0-160-generic-x86_64-with-glibc2.39", + "python": "CPython 3.11.14", + "startedAt": "2025-12-19T03:28:09.455870Z", + "args": [ + "configs/test/OLMoE-7B-500GB-test.yaml", + "--run_name=olmoe-7b-29b-2ep", + "--save_overwrite" + ], + "program": "/data/users/xandi281/work/Hi-OLMoE/OLMo/scripts/train.py", + "codePath": "OLMo/scripts/train.py", + "codePathLocal": "scripts/train.py", + "git": { + "remote": "git@github.com:AverageMetaheuristicsEnjoyer/Hi-OLMoE.git", + "commit": "9b3ad56c62f3b7b0e435efc757c49704db6e6333" + }, + "email": "alexmyasnikov451@gmail.com", + "root": "runs/olmoe-7b-29b-2ep/wandb", + "host": "ubuntu-gpu-pod-final", + "executable": "/data/users/xandi281/work/Hi-OLMoE/venv/bin/python3", + "cpu_count": 96, + "cpu_count_logical": 192, + "gpu": "NVIDIA H200", + "gpu_count": 8, + "disk": { + "/": { + "total": "422146228224", + "used": "65563865088" + } + }, + "memory": { + "total": "2163971334144" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-28548a6f-a458-cff0-d4d9-cd8992f7bfc8" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-390840f6-de3c-1ecc-1353-cf4d01ab89e2" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-bbba09e7-ef87-452c-7d34-cb2a03510f2c" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-722903bc-18d9-e3ef-0146-caff9c35e5bf" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-f09af96c-4a99-e9b3-2e01-4d03d1a959dd" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-0aa55090-5ce5-56e2-62cc-36bc2ae4fb77" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-8a81a780-cddb-5b47-490f-871873b601ec" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-81dfcc32-ea97-c0f1-778e-ff44c4992c1a" + } + ], + "cudaVersion": "13.0", + "writerId": "hkx9xgphpmzezm5xyyncldepfa9kpirs" +} \ No newline at end of file diff --git a/wandb/wandb/run-20251219_032809-fhag09gx/files/wandb-summary.json b/wandb/wandb/run-20251219_032809-fhag09gx/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..bf25c54a2cb5f8454c1408f991d6f054ec2f4abe --- /dev/null +++ b/wandb/wandb/run-20251219_032809-fhag09gx/files/wandb-summary.json @@ -0,0 +1 @@ +{"optim/grad/transformer.blocks.10.ffn.experts.mlp.w2.norm":0.002134507754817605,"optim/exp_avg_sq/transformer.blocks.14.ffn.experts.mlp.w2.norm":4.97178243019647e-10,"optim/exp_avg_sq/transformer.blocks.10.attn_norm.weight.norm":3.936448944585891e-09,"optim/grad/transformer.blocks.11.k_norm.weight.norm":0.0001003930447041057,"optim/grad/transformer.blocks.6.ffn.experts.mlp.w1.avg":4.00580055237576e-11,"optim/grad/transformer.blocks.0.q_norm.weight.avg":-2.4827983224895434e-07,"optim/exp_avg/transformer.blocks.14.attn_norm.weight.min":5.014762494148783e-10,"optim/exp_avg/transformer.blocks.7.ff_norm.weight.max":1.251396861334797e-05,"optim/grad/transformer.blocks.2.ff_norm.weight.norm":0.0001109917284338735,"optim/exp_avg_sq/transformer.blocks.13.k_norm.weight.max":3.0040825382826597e-10,"optim/exp_avg_sq/transformer.blocks.7.att_proj.weight.min":9.373863487846106e-17,"optim/exp_avg_sq/transformer.blocks.10.q_norm.weight.avg":7.871876761544883e-12,"optim/exp_avg/transformer.blocks.14.ffn.experts.mlp.w2.avg":1.9140321272370642e-11,"optim/exp_avg_sq/transformer.blocks.7.attn_out.weight.norm":1.0678125050844756e-08,"optim/exp_avg/transformer.blocks.1.ffn.experts.mlp.w2.min":1.1694560386266498e-16,"optim/exp_avg_sq/transformer.blocks.6.attn_out.weight.norm":1.635774893316011e-08,"optim/exp_avg_sq/transformer.blocks.0.ff_norm.weight.min":3.424243310776176e-13,"optim/param/transformer.blocks.10.ffn.router.layer.weight.max":0.15177081525325775,"optim/grad/transformer.blocks.9.k_norm.weight.avg":-1.5434592626206722e-07,"optim/grad/transformer.blocks.13.attn_out.weight.max":1.9194019841961563e-05,"optim/grad/transformer.blocks.4.ff_norm.weight.avg":1.8858056449744254e-08,"optim/grad/transformer.blocks.5.att_proj.weight.max":0.0010559340007603168,"optim/param/transformer.blocks.11.attn_out.weight.max":0.1946573555469513,"optim/exp_avg/transformer.blocks.1.attn_norm.weight.norm":6.975207361392677e-05,"optim/exp_avg/transformer.blocks.10.q_norm.weight.max":3.97797157347668e-06,"optim/exp_avg/transformer.blocks.3.att_proj.weight.avg":-4.478963380738499e-11,"optim/grad/transformer.blocks.2.ff_norm.weight.max":0.0001417376333847642,"optim/grad/transformer.blocks.3.att_proj.weight.max":0.0009020568686537445,"optim/param/transformer.blocks.15.ffn.experts.mlp.w1.avg":7.582783837278839e-06,"optim/grad/transformer.blocks.9.q_norm.weight.avg":-1.66220218034141e-07,"optim/grad/transformer.blocks.7.attn_out.weight.min":0,"optim/grad/transformer.blocks.10.ffn.router.layer.weight.max":8.479997632093728e-05,"optim/grad/transformer.blocks.13.attn_norm.weight.avg":-5.4231023227657715e-08,"optim/exp_avg_sq/transformer.blocks.3.ffn.router.layer.weight.avg":5.997164570503699e-11,"optim/exp_avg_sq/transformer.ff_out.weight.norm":3.841954821837135e-06,"optim/grad/transformer.blocks.10.ff_norm.weight.min":2.0245352061465383e-09,"optim/exp_avg_sq/transformer.blocks.4.ff_norm.weight.max":2.542062338406481e-09,"optim/exp_avg_sq/transformer.blocks.9.ff_norm.weight.norm":5.837925698415347e-10,"optim/exp_avg_sq/transformer.blocks.5.q_norm.weight.norm":4.507290540800568e-08,"eval/dolma_reddit-validation/Perplexity":47.7489013671875,"optim/exp_avg/transformer.blocks.8.ffn.experts.mlp.w2.max":9.091738775168778e-07,"optim/exp_avg/transformer.blocks.7.ffn.experts.mlp.w1.min":1.879771091930528e-15,"optim/param/transformer.blocks.1.q_norm.weight.norm":40.4294319152832,"eval/downstream/hellaswag_len_norm":0.3042222559452057,"optim/param/transformer.blocks.10.ffn.experts.mlp.w2.min":3.400262276603838e-12,"optim/exp_avg_sq/transformer.blocks.2.attn_out.weight.avg":6.314135846119395e-12,"optim/exp_avg/transformer.blocks.7.att_proj.weight.min":1.339655112102961e-14,"optim/exp_avg/transformer.blocks.8.ffn.experts.mlp.w1.max":8.608717507740948e-06,"optim/exp_avg/transformer.blocks.7.attn_norm.weight.min":2.200200716062639e-10,"optim/param/transformer.blocks.8.attn_out.weight.min":5.808805436657849e-09,"optim/exp_avg/transformer.blocks.3.attn_out.weight.max":9.433506420464255e-06,"optim/grad/transformer.blocks.2.q_norm.weight.avg":-1.9839737319671258e-07,"optim/grad/transformer.blocks.14.ffn.experts.mlp.w1.avg":-5.55735535714974e-12,"optim/grad/transformer.blocks.15.ff_norm.weight.avg":-3.7630314864145475e-07,"optim/grad/transformer.blocks.10.ffn.experts.mlp.w1.max":8.740108569327276e-06,"optim/exp_avg/transformer.blocks.15.ff_norm.weight.avg":-3.2141878136826563e-07,"optim/exp_avg/transformer.blocks.8.ff_norm.weight.min":5.977673911417014e-11,"optim/param/transformer.blocks.14.attn_out.weight.norm":43.330684661865234,"optim/param/transformer.blocks.15.q_norm.weight.avg":0.8996568918228149,"optim/exp_avg_sq/transformer.blocks.13.attn_out.weight.norm":3.2016844730975436e-09,"optim/exp_avg_sq/transformer.blocks.15.ffn.experts.mlp.w1.avg":4.3942575137434145e-14,"optim/exp_avg/transformer.blocks.15.ffn.router.layer.weight.avg":4.0854097882458973e-10,"optim/grad/transformer.blocks.8.ff_norm.weight.min":1.4406396076083183e-09,"optim/grad/transformer.blocks.15.k_norm.weight.norm":7.970231672516093e-05,"optim/exp_avg_sq/transformer.blocks.5.attn_out.weight.max":1.0886117701724984e-10,"optim/exp_avg/transformer.blocks.2.ffn.experts.mlp.w1.norm":0.0006477723945863545,"optim/exp_avg_sq/transformer.wte.weight.avg":1.6829305960960883e-13,"optim/exp_avg_sq/transformer.blocks.7.attn_out.weight.min":1.541821954397643e-13,"optim/grad/transformer.blocks.6.attn_norm.weight.max":0.00010246352758258581,"optim/exp_avg_sq/transformer.blocks.5.k_norm.weight.max":2.359201722867965e-08,"optim/exp_avg/transformer.blocks.13.attn_out.weight.max":4.176680249656783e-06,"optim/grad/transformer.blocks.6.ffn.experts.mlp.w2.avg":7.254809947232488e-11,"_timestamp":1.766671046819392e+09,"optim/param/transformer.blocks.6.ffn.experts.mlp.w2.norm":319.769775390625,"optim/param/transformer.blocks.5.ff_norm.weight.min":0.6237497329711914,"optim/param/transformer.blocks.13.k_norm.weight.max":1.459140419960022,"optim/param/transformer.blocks.14.ffn.experts.mlp.w1.avg":1.1293237776044407e-06,"optim/total_grad_norm":0.05781364068388939,"optim/grad/transformer.blocks.11.q_norm.weight.max":2.8166316042188555e-05,"optim/exp_avg_sq/transformer.blocks.8.ffn.experts.mlp.w1.max":1.7302269617047727e-09,"optim/exp_avg/transformer.blocks.8.k_norm.weight.min":5.943697617416532e-11,"optim/param/transformer.blocks.15.ff_norm.weight.max":1.1424285173416138,"optim/param/transformer.blocks.5.q_norm.weight.min":0.5038654804229736,"optim/exp_avg/transformer.blocks.7.ffn.router.layer.weight.norm":0.0006341185653582215,"optim/grad/transformer.blocks.14.attn_norm.weight.min":1.3387762010097504e-09,"optim/param/transformer.blocks.10.attn_norm.weight.min":0.545894980430603,"optim/grad/transformer.blocks.12.attn_out.weight.min":2.2737367544323206e-13,"optim/exp_avg_sq/transformer.blocks.12.ffn.router.layer.weight.avg":4.4607845195443474e-11,"optim/exp_avg/transformer.blocks.6.att_proj.weight.norm":0.002178693190217018,"optim/param/transformer.blocks.7.ffn.router.layer.weight.max":0.12984928488731384,"optim/exp_avg_sq/transformer.blocks.8.ffn.experts.mlp.w2.min":1.1055656139646788e-15,"optim/exp_avg_sq/transformer.blocks.14.attn_out.weight.max":4.559910088408614e-11,"optim/exp_avg/transformer.blocks.4.ffn.experts.mlp.w1.avg":5.944614592245934e-12,"optim/exp_avg_sq/transformer.blocks.7.k_norm.weight.norm":2.3512238822576137e-09,"optim/exp_avg_sq/transformer.blocks.14.ff_norm.weight.max":8.081746383226118e-10,"optim/exp_avg/transformer.blocks.0.q_norm.weight.max":3.0111184514680644e-06,"optim/exp_avg_sq/transformer.blocks.9.k_norm.weight.max":4.70374317274036e-09,"optim/exp_avg_sq/transformer.blocks.12.ffn.experts.mlp.w2.max":1.1911045066925752e-12,"optim/exp_avg_sq/transformer.blocks.13.q_norm.weight.avg":3.763632201031486e-12,"optim/exp_avg/transformer.blocks.14.ffn.experts.mlp.w2.min":1.3684198427719486e-16,"optim/exp_avg_sq/transformer.blocks.15.att_proj.weight.avg":1.1326622148880028e-12,"optim/grad/transformer.blocks.7.attn_out.weight.norm":0.003981711342930794,"optim/grad/transformer.blocks.1.k_norm.weight.norm":0.00010273762745782733,"optim/exp_avg/transformer.blocks.14.ffn.router.layer.weight.max":2.9693394026253372e-05,"optim/grad/transformer.blocks.14.ffn.experts.mlp.w2.avg":1.6965628901743912e-10,"optim/exp_avg/transformer.blocks.2.attn_norm.weight.max":0.00010559555084910244,"optim/exp_avg/transformer.blocks.4.k_norm.weight.min":1.3485895733467146e-10,"optim/exp_avg_sq/transformer.blocks.0.q_norm.weight.min":2.536037650496467e-14,"optim/exp_avg/transformer.blocks.14.ffn.router.layer.weight.min":1.4155458923081898e-11,"optim/param/transformer.blocks.12.att_proj.weight.min":1.893492251880957e-09,"_runtime":556201.82222337,"optim/grad/transformer.blocks.8.att_proj.weight.min":0,"optim/exp_avg/transformer.blocks.2.k_norm.weight.avg":-1.6686091441897588e-07,"optim/grad/transformer.blocks.8.ff_norm.weight.norm":0.00010047874093288556,"optim/grad/transformer.blocks.6.ffn.router.layer.weight.avg":9.968026404294505e-11,"optim/param/transformer.blocks.1.attn_out.weight.norm":33.672115325927734,"optim/exp_avg_sq/transformer.blocks.3.ffn.router.layer.weight.min":6.367086545500111e-12,"optim/param/transformer.blocks.7.ff_norm.weight.max":1.0841354131698608,"optim/grad/transformer.blocks.8.attn_norm.weight.max":7.456469757016748e-05,"optim/grad/transformer.blocks.12.ffn.router.layer.weight.max":0.0001039386261254549,"optim/grad/transformer.blocks.2.attn_norm.weight.norm":0.00030420348048210144,"optim/exp_avg/transformer.blocks.0.attn_norm.weight.avg":4.946172111885971e-07,"optim/exp_avg_sq/transformer.blocks.14.attn_norm.weight.norm":1.6046309836426076e-09,"optim/grad/transformer.blocks.14.ffn.experts.mlp.w2.min":0,"optim/grad/transformer.blocks.5.q_norm.weight.norm":0.00013831042451784015,"optim/exp_avg/transformer.blocks.0.ff_norm.weight.norm":2.413511901977472e-05,"optim/exp_avg_sq/transformer.blocks.2.ff_norm.weight.avg":8.874304936734578e-12,"optim/exp_avg/transformer.blocks.9.att_proj.weight.min":3.898860807772728e-14,"optim/grad/transformer.blocks.14.ffn.router.layer.weight.min":4.001776687800884e-11,"optim/param/transformer.blocks.9.attn_out.weight.min":2.165051471436641e-09,"optim/exp_avg/transformer.blocks.3.ffn.experts.mlp.w2.norm":0.000688136788085103,"optim/exp_avg/transformer.blocks.2.q_norm.weight.min":3.0270998381543013e-12,"optim/grad/transformer.blocks.0.q_norm.weight.norm":7.503155939048156e-05,"optim/param/transformer.blocks.10.ffn.experts.mlp.w1.max":0.17350876331329346,"optim/param/transformer.blocks.5.attn_out.weight.norm":40.049503326416016,"optim/exp_avg/transformer.blocks.10.ff_norm.weight.max":3.053983164136298e-06,"optim/exp_avg_sq/transformer.blocks.13.ffn.router.layer.weight.min":5.006980073607448e-12,"optim/exp_avg/transformer.blocks.0.ffn.router.layer.weight.norm":0.0004128684231545776,"optim/exp_avg/transformer.blocks.3.ff_norm.weight.avg":-2.0207632189794822e-07,"optim/param/transformer.blocks.13.attn_norm.weight.norm":30.880342483520508,"optim/param/transformer.blocks.1.att_proj.weight.min":2.596550974232059e-09,"optim/param/transformer.blocks.11.ff_norm.weight.min":0.5706869959831238,"optim/exp_avg_sq/transformer.blocks.14.attn_norm.weight.avg":7.446808794608906e-12,"optim/exp_avg/transformer.blocks.4.attn_out.weight.avg":-2.55300676432757e-11,"optim/grad/transformer.blocks.8.att_proj.weight.norm":0.009489783085882664,"optim/param/transformer.blocks.3.ffn.experts.mlp.w1.max":0.17426112294197083,"optim/param/transformer.blocks.12.attn_out.weight.avg":1.6192843759199604e-06,"optim/param/transformer.blocks.14.attn_out.weight.max":0.21992440521717072,"optim/grad/transformer.blocks.9.ffn.router.layer.weight.min":1.7189449863508344e-10,"optim/param/transformer.blocks.9.att_proj.weight.norm":78.58779907226562,"optim/param/transformer.blocks.14.attn_norm.weight.avg":0.65340656042099,"optim/grad/transformer.blocks.4.k_norm.weight.norm":0.00023868509742897004,"optim/exp_avg_sq/transformer.blocks.12.k_norm.weight.max":3.706525364410851e-10,"optim/exp_avg_sq/transformer.blocks.4.q_norm.weight.max":5.825854909602413e-09,"optim/exp_avg/transformer.blocks.2.att_proj.weight.min":6.150318427287915e-14,"optim/exp_avg_sq/transformer.blocks.8.att_proj.weight.max":1.2841981344990927e-07,"optim/exp_avg_sq/transformer.blocks.14.q_norm.weight.avg":2.3602816749679345e-12,"optim/grad/transformer.blocks.7.ff_norm.weight.max":2.3500804672949016e-05,"optim/exp_avg/transformer.blocks.15.q_norm.weight.min":5.4210611694083255e-12,"optim/param/transformer.blocks.1.attn_norm.weight.norm":17.646059036254883,"optim/exp_avg/transformer.wte.weight.max":3.7255285860737786e-05,"optim/exp_avg_sq/transformer.blocks.12.ffn.experts.mlp.w2.min":8.236386913445246e-17,"optim/exp_avg_sq/transformer.blocks.5.ffn.experts.mlp.w1.norm":6.577311140176789e-09,"optim/param/transformer.blocks.12.ff_norm.weight.norm":51.27845001220703,"optim/grad/transformer.blocks.0.ff_norm.weight.avg":-9.064650896561943e-08,"optim/grad/transformer.blocks.1.ffn.experts.mlp.w2.avg":-2.7120028445182243e-10,"optim/exp_avg/transformer.blocks.6.ffn.router.layer.weight.min":1.3568842230360367e-11,"optim/grad/transformer.blocks.4.ffn.router.layer.weight.norm":0.0030723633244633675,"optim/grad/transformer.blocks.1.att_proj.weight.min":0,"optim/exp_avg_sq/transformer.blocks.4.attn_norm.weight.avg":1.5644634199230012e-10,"optim/grad/transformer.blocks.10.ffn.experts.mlp.w1.min":0,"optim/exp_avg_sq/transformer.blocks.8.k_norm.weight.min":5.6362457459731025e-15,"optim/param/transformer.blocks.13.att_proj.weight.avg":2.406337671345682e-06,"optim/exp_avg_sq/transformer.blocks.0.ffn.experts.mlp.w1.min":2.0048572961843213e-18,"optim/grad/transformer.blocks.6.attn_norm.weight.avg":3.030537527592969e-07,"optim/param/transformer.blocks.0.k_norm.weight.max":1.197163462638855,"optim/exp_avg_sq/transformer.blocks.2.att_proj.weight.min":2.8081391073955016e-15,"optim/exp_avg/transformer.blocks.15.att_proj.weight.min":9.89204547538914e-15,"optim/exp_avg_sq/transformer.blocks.11.q_norm.weight.avg":4.491869549927419e-12,"optim/exp_avg/transformer.blocks.15.ffn.experts.mlp.w1.min":4.799192329145116e-16,"optim/exp_avg_sq/transformer.blocks.11.att_proj.weight.norm":7.590715256355907e-08,"optim/grad/transformer.blocks.13.q_norm.weight.min":2.5693225325085223e-11,"optim/exp_avg_sq/transformer.blocks.11.ffn.router.layer.weight.max":7.461970596978063e-09,"optim/exp_avg/transformer.blocks.6.k_norm.weight.min":1.677782218711954e-10,"optim/param/transformer.blocks.2.ffn.router.layer.weight.min":5.588447606896807e-07,"optim/exp_avg_sq/transformer.blocks.2.attn_out.weight.min":5.175365344611826e-13,"optim/grad/transformer.blocks.6.att_proj.weight.min":0,"optim/exp_avg_sq/transformer.blocks.8.ffn.experts.mlp.w2.norm":6.230878590862687e-10,"optim/grad/transformer.blocks.12.ffn.experts.mlp.w1.norm":0.0024244352243840694,"optim/exp_avg/transformer.blocks.6.attn_out.weight.min":2.7783401664385754e-13,"optim/param/transformer.blocks.11.ffn.router.layer.weight.norm":7.711306571960449,"optim/grad/transformer.blocks.1.ffn.experts.mlp.w1.avg":-1.506083452396112e-12,"optim/param/transformer.blocks.9.ff_norm.weight.avg":1.0490204095840454,"optim/param/transformer.blocks.2.att_proj.weight.max":0.17734353244304657,"optim/exp_avg_sq/transformer.blocks.7.ffn.experts.mlp.w2.norm":6.824493192780778e-10,"optim/exp_avg_sq/transformer.blocks.7.attn_out.weight.avg":3.775126045102439e-12,"optim/exp_avg/transformer.blocks.13.ffn.experts.mlp.w2.max":8.167920100277115e-07,"optim/grad/transformer.blocks.5.k_norm.weight.max":2.8607999411178753e-05,"optim/grad/transformer.blocks.4.q_norm.weight.norm":0.00020444147230591625,"optim/param/transformer.blocks.15.ffn.router.layer.weight.min":8.720233779513364e-08,"optim/exp_avg_sq/transformer.blocks.7.k_norm.weight.min":7.043459459501282e-17,"optim/param/transformer.blocks.7.q_norm.weight.max":1.322571039199829,"optim/param/transformer.blocks.14.ffn.experts.mlp.w1.min":1.3333079085242616e-10,"optim/param/transformer.blocks.13.ffn.experts.mlp.w2.max":0.5530311465263367,"optim/param/transformer.blocks.6.att_proj.weight.max":0.17973461747169495,"optim/param/transformer.blocks.13.q_norm.weight.avg":0.9295116662979126,"optim/grad/transformer.blocks.2.attn_norm.weight.min":6.9521775003522635e-09,"optim/exp_avg_sq/transformer.blocks.3.attn_norm.weight.norm":8.562973619063996e-08,"optim/exp_avg/transformer.blocks.11.attn_norm.weight.min":2.962461387312487e-10,"optim/grad/transformer.blocks.13.att_proj.weight.avg":-1.3135793841545507e-10,"optim/param/transformer.blocks.10.att_proj.weight.max":0.16839705407619476,"optim/exp_avg_sq/transformer.blocks.15.ff_norm.weight.max":2.584336744604343e-09,"optim/grad/transformer.blocks.5.ffn.router.layer.weight.max":0.0003935681888833642,"optim/exp_avg/transformer.blocks.4.att_proj.weight.min":1.0158810878830356e-14,"optim/exp_avg/transformer.blocks.6.q_norm.weight.norm":2.8580161597346887e-05,"optim/grad/transformer.blocks.15.ffn.experts.mlp.w1.avg":8.721477533224498e-12,"optim/param/transformer.blocks.12.ffn.experts.mlp.w1.max":0.22121575474739075,"optim/grad/transformer.blocks.13.q_norm.weight.norm":8.103875734377652e-05,"optim/param/transformer.blocks.5.ffn.experts.mlp.w2.max":0.25455451011657715,"optim/exp_avg/transformer.blocks.10.k_norm.weight.max":3.903455308318371e-06,"optim/exp_avg_sq/transformer.blocks.6.q_norm.weight.norm":3.726044806029449e-09,"optim/exp_avg_sq/transformer.blocks.5.attn_norm.weight.avg":1.2282730388335494e-10,"optim/exp_avg/transformer.blocks.13.att_proj.weight.max":1.595779394847341e-05,"optim/exp_avg/transformer.blocks.4.att_proj.weight.norm":0.002890047151595354,"optim/param/transformer.blocks.10.q_norm.weight.avg":0.963689386844635,"optim/param/transformer.blocks.9.attn_out.weight.max":0.1569490134716034,"optim/exp_avg/transformer.blocks.2.q_norm.weight.avg":-1.674747238666896e-07,"optim/grad/transformer.blocks.6.att_proj.weight.norm":0.012946438044309616,"optim/exp_avg/transformer.blocks.15.ffn.experts.mlp.w2.norm":0.0003590585838537663,"optim/exp_avg/transformer.blocks.10.ffn.router.layer.weight.avg":-9.25441945298644e-11,"optim/param/transformer.blocks.11.ff_norm.weight.avg":1.1144671440124512,"optim/param/transformer.blocks.5.k_norm.weight.norm":44.92311096191406,"optim/exp_avg/transformer.blocks.9.attn_out.weight.avg":3.6680958270807196e-10,"optim/param/transformer.blocks.7.q_norm.weight.min":0.3561497926712036,"optim/param/transformer.blocks.13.ffn.experts.mlp.w1.min":4.761387950580342e-10,"optim/param/transformer.blocks.15.k_norm.weight.max":1.244707703590393,"optim/grad/transformer.blocks.5.attn_norm.weight.max":0.00023505472927354276,"optim/exp_avg_sq/transformer.blocks.5.att_proj.weight.avg":2.305541348113227e-11,"optim/param/transformer.blocks.14.att_proj.weight.avg":-5.241354756435612e-06,"optim/param/transformer.blocks.1.ff_norm.weight.max":1.0463756322860718,"optim/exp_avg_sq/transformer.blocks.15.att_proj.weight.min":6.793508110049316e-16,"optim/param/transformer.blocks.14.ffn.experts.mlp.w2.max":1.167607069015503,"optim/exp_avg/transformer.blocks.9.attn_out.weight.norm":0.0008014551713131368,"optim/param/transformer.blocks.2.ffn.router.layer.weight.avg":0.00010887191456276923,"optim/exp_avg/transformer.blocks.0.ffn.experts.mlp.w1.avg":1.96993914236554e-11,"optim/grad/transformer.blocks.8.attn_out.weight.avg":5.606936026580911e-10,"optim/exp_avg/transformer.blocks.6.att_proj.weight.avg":1.025962093059718e-10,"optim/grad/transformer.blocks.9.attn_out.weight.norm":0.00358216161839664,"optim/param/transformer.blocks.3.attn_out.weight.avg":-1.4440742234000936e-05,"optim/exp_avg/transformer.blocks.4.ffn.router.layer.weight.min":8.555127960219178e-12,"optim/exp_avg_sq/transformer.blocks.11.k_norm.weight.avg":4.6230632169685926e-12,"optim/param/transformer.blocks.12.attn_out.weight.norm":47.407440185546875,"optim/grad/transformer.blocks.7.ff_norm.weight.min":2.089109329972416e-09,"optim/exp_avg/transformer.blocks.14.ffn.experts.mlp.w2.max":1.315386725764256e-06,"optim/grad/transformer.blocks.9.k_norm.weight.norm":0.00013952417066320777,"optim/param/transformer.blocks.11.ffn.router.layer.weight.min":2.1339597822134238e-07,"optim/grad/transformer.blocks.8.ffn.experts.mlp.w2.norm":0.0024622222408652306,"optim/exp_avg/transformer.blocks.13.attn_out.weight.min":5.705422164445709e-14,"optim/exp_avg/transformer.blocks.8.k_norm.weight.avg":-1.0989266030492217e-07,"optim/grad/transformer.blocks.0.attn_norm.weight.max":8.348764094989747e-05,"optim/grad/transformer.blocks.12.attn_out.weight.max":8.642542525194585e-06,"optim/exp_avg_sq/transformer.blocks.3.attn_norm.weight.min":8.916522034607688e-12,"optim/param/transformer.blocks.8.attn_norm.weight.min":0.5429130792617798,"optim/exp_avg_sq/transformer.blocks.6.k_norm.weight.norm":3.87893139830453e-09,"optim/exp_avg_sq/transformer.blocks.6.ffn.experts.mlp.w2.norm":5.838383665413005e-10,"optim/exp_avg/transformer.blocks.6.attn_out.weight.max":5.501494342752267e-06,"optim/grad/transformer.blocks.6.ffn.experts.mlp.w1.max":3.0591461836593226e-05,"optim/grad/transformer.blocks.1.ff_norm.weight.norm":9.529211092740297e-05,"optim/exp_avg_sq/transformer.blocks.10.ffn.experts.mlp.w1.max":1.906210952973275e-11,"optim/exp_avg_sq/transformer.blocks.14.ffn.experts.mlp.w1.avg":4.1803989740338277e-14,"optim/exp_avg/transformer.blocks.0.ffn.router.layer.weight.avg":7.32191418606476e-10,"optim/param/transformer.blocks.3.attn_out.weight.norm":36.54835510253906,"optim/param/transformer.blocks.0.k_norm.weight.min":0.6100175380706787,"optim/exp_avg/transformer.blocks.4.k_norm.weight.avg":-1.8850833782835252e-07,"optim/grad/transformer.blocks.7.k_norm.weight.min":1.1553780154827109e-10,"optim/param/transformer.blocks.0.attn_norm.weight.avg":0.32715025544166565,"optim/grad/transformer.blocks.5.attn_norm.weight.norm":0.00041324805351905525,"optim/grad/transformer.blocks.13.q_norm.weight.max":2.3635817342437804e-05,"optim/exp_avg/transformer.blocks.2.k_norm.weight.min":4.3268005844865343e-10,"optim/exp_avg_sq/transformer.blocks.15.k_norm.weight.max":1.557611817304405e-10,"optim/exp_avg_sq/transformer.blocks.5.ffn.router.layer.weight.avg":7.160372295089701e-11,"optim/exp_avg_sq/transformer.blocks.11.ff_norm.weight.min":1.402313486685569e-12,"optim/exp_avg/transformer.blocks.7.attn_norm.weight.max":1.8178203390561976e-05,"optim/exp_avg/transformer.blocks.12.q_norm.weight.avg":-1.389312274113763e-07,"optim/exp_avg/transformer.blocks.6.k_norm.weight.avg":-1.9133111095470667e-07,"optim/exp_avg_sq/transformer.blocks.6.ff_norm.weight.min":1.8856719571219704e-12,"optim/exp_avg/transformer.blocks.15.attn_norm.weight.min":7.387711969952804e-11,"optim/param/transformer.blocks.15.att_proj.weight.min":7.232025533987496e-10,"optim/exp_avg/transformer.blocks.13.att_proj.weight.avg":3.848832363928523e-11,"optim/grad/transformer.blocks.14.att_proj.weight.min":0,"optim/exp_avg_sq/transformer.blocks.4.attn_out.weight.avg":7.981667410339455e-12,"optim/grad/transformer.blocks.1.attn_out.weight.norm":0.004867095500230789,"optim/grad/transformer.blocks.15.ffn.experts.mlp.w2.max":4.25326788899838e-06,"optim/exp_avg_sq/transformer.blocks.15.k_norm.weight.norm":5.226851729212001e-10,"optim/grad/transformer.blocks.11.attn_out.weight.norm":0.0028472214471548796,"optim/exp_avg_sq/transformer.blocks.7.ffn.router.layer.weight.max":1.965912055368335e-08,"optim/param/transformer.blocks.5.ffn.experts.mlp.w1.avg":6.03335392952431e-05,"optim/grad/transformer.blocks.1.attn_out.weight.avg":1.1668688237875813e-09,"optim/exp_avg/transformer.blocks.8.ffn.experts.mlp.w2.avg":2.57919241519744e-11,"optim/grad/transformer.blocks.5.ffn.experts.mlp.w2.avg":1.1215288420185487e-10,"optim/exp_avg_sq/transformer.blocks.6.k_norm.weight.avg":1.3037879803556862e-11,"optim/param/transformer.blocks.11.q_norm.weight.min":0.5306609869003296,"optim/grad/transformer.blocks.14.ffn.router.layer.weight.avg":8.129674711199186e-11,"optim/param/transformer.blocks.15.att_proj.weight.norm":75.65351104736328,"optim/exp_avg/transformer.blocks.5.ffn.experts.mlp.w1.norm":0.0007549027213826776,"optim/exp_avg_sq/transformer.blocks.1.ff_norm.weight.norm":2.5765849454018053e-09,"optim/exp_avg_sq/transformer.blocks.12.attn_norm.weight.avg":1.1401164734525793e-11,"optim/exp_avg/transformer.blocks.4.ff_norm.weight.max":9.305464118369855e-06,"optim/exp_avg_sq/transformer.blocks.6.attn_norm.weight.avg":6.874568969639228e-11,"eval/wikitext_103-validation/CrossEntropyLoss":3.518427848815918,"optim/grad/transformer.blocks.7.ffn.experts.mlp.w2.avg":2.3043325581006968e-10,"optim/exp_avg_sq/transformer.blocks.1.ff_norm.weight.avg":5.632468016297798e-12,"optim/exp_avg_sq/transformer.blocks.7.attn_norm.weight.avg":5.96150490084213e-11,"optim/exp_avg/transformer.blocks.13.ffn.experts.mlp.w2.min":1.0618439966888983e-15,"optim/exp_avg_sq/transformer.blocks.14.ffn.experts.mlp.w2.avg":2.2154038315218977e-14,"optim/exp_avg_sq/transformer.blocks.13.attn_out.weight.max":7.240150839971093e-11,"optim/grad/transformer.blocks.2.attn_norm.weight.avg":8.944422091872184e-08,"optim/grad/transformer.blocks.3.ffn.router.layer.weight.norm":0.0027392811607569456,"optim/exp_avg_sq/transformer.blocks.5.k_norm.weight.min":1.2615212355098458e-14,"optim/exp_avg/transformer.blocks.7.ffn.experts.mlp.w2.min":5.423148561828155e-16,"optim/param/transformer.blocks.15.k_norm.weight.norm":40.59012222290039,"optim/grad/transformer.blocks.12.ff_norm.weight.avg":-1.00448147577481e-08,"optim/param/transformer.blocks.5.q_norm.weight.max":1.3322418928146362,"optim/exp_avg/transformer.blocks.3.att_proj.weight.max":0.00012445788888726383,"optim/exp_avg_sq/transformer.blocks.10.k_norm.weight.min":5.348966051582334e-15,"optim/param/transformer.blocks.10.ffn.router.layer.weight.min":2.408138186638098e-07,"optim/param/transformer.blocks.14.k_norm.weight.avg":0.9194515943527222,"optim/exp_avg/transformer.blocks.4.k_norm.weight.norm":4.820161484531127e-05,"optim/grad/transformer.blocks.10.att_proj.weight.max":0.00014601432485505939,"optim/param/transformer.blocks.7.att_proj.weight.max":0.1850728541612625,"optim/grad/transformer.blocks.1.k_norm.weight.min":1.8098944565281272e-10,"optim/grad/transformer.blocks.11.q_norm.weight.min":1.9504113879520446e-09,"optim/grad/transformer.blocks.14.ff_norm.weight.min":7.330527296289802e-10,"optim/exp_avg_sq/transformer.blocks.15.attn_norm.weight.avg":1.249843457895583e-11,"optim/exp_avg/transformer.blocks.13.ff_norm.weight.max":2.4981154638226144e-06,"optim/param/transformer.blocks.13.ffn.experts.mlp.w1.avg":2.783539457595907e-06,"optim/grad/transformer.blocks.6.attn_out.weight.avg":1.4971025530385873e-09,"optim/clipping_rate":0,"optim/grad/transformer.blocks.0.ffn.experts.mlp.w1.min":0,"optim/param/transformer.blocks.8.att_proj.weight.avg":5.0288072088733315e-06,"optim/exp_avg/transformer.blocks.13.attn_norm.weight.avg":3.3154275769220476e-08,"optim/exp_avg/transformer.blocks.6.ff_norm.weight.norm":3.1130563002079725e-05,"optim/grad/transformer.blocks.5.ffn.experts.mlp.w1.avg":-4.479339815732786e-11,"optim/exp_avg/transformer.blocks.1.ff_norm.weight.max":6.572205165866762e-06,"optim/param/transformer.blocks.7.attn_out.weight.norm":41.05770492553711,"optim/param/transformer.blocks.13.ff_norm.weight.norm":51.58472442626953,"optim/grad/transformer.blocks.8.ffn.experts.mlp.w2.max":6.043841040082043e-06,"optim/param/transformer.blocks.15.attn_out.weight.avg":1.2003296433249488e-05,"optim/grad/transformer.blocks.7.ffn.experts.mlp.w1.avg":-3.773899942549619e-11,"optim/param/transformer.blocks.5.attn_out.weight.avg":-4.110345798835624e-06,"optim/exp_avg_sq/transformer.blocks.6.att_proj.weight.avg":1.3801480791553189e-11,"optim/exp_avg/transformer.blocks.10.attn_norm.weight.min":9.955548607720743e-10,"optim/grad/transformer.blocks.13.ffn.router.layer.weight.max":0.00013837240112479776,"optim/exp_avg_sq/transformer.blocks.3.att_proj.weight.max":1.8838682080968283e-07,"eval/dolma_wiki-validation/CrossEntropyLoss":3.479590654373169,"optim/grad/transformer.blocks.6.attn_norm.weight.min":8.976712706498802e-10,"optim/param/transformer.blocks.7.ffn.experts.mlp.w1.min":9.572141967950643e-10,"optim/exp_avg/transformer.blocks.1.ffn.experts.mlp.w1.min":1.6998998339328147e-17,"optim/param/transformer.blocks.2.ff_norm.weight.min":0.43046054244041443,"optim/exp_avg_sq/transformer.blocks.15.attn_out.weight.min":4.5936632996805907e-14,"optim/exp_avg_sq/transformer.blocks.2.q_norm.weight.avg":6.916308237930879e-12,"optim/exp_avg_sq/transformer.blocks.9.att_proj.weight.avg":4.7874694659999495e-12,"optim/param/transformer.blocks.8.ff_norm.weight.norm":45.54121780395508,"optim/param/transformer.blocks.11.attn_out.weight.norm":44.50960159301758,"optim/param/transformer.blocks.1.att_proj.weight.norm":76.62825012207031,"optim/exp_avg/transformer.blocks.9.ffn.router.layer.weight.max":2.475795554346405e-05,"optim/exp_avg/transformer.blocks.2.attn_out.weight.avg":-1.1685083456391965e-11,"optim/exp_avg/transformer.blocks.13.q_norm.weight.max":4.468375664146151e-06,"optim/param/transformer.blocks.13.attn_norm.weight.min":0.5127901434898376,"optim/param/transformer.blocks.13.attn_out.weight.max":0.21238622069358826,"optim/exp_avg_sq/transformer.blocks.13.q_norm.weight.norm":6.379686223745296e-10,"optim/param/transformer.blocks.7.ff_norm.weight.norm":44.681671142578125,"optim/grad/transformer.ln_f.weight.max":0.00018427101895213127,"optim/grad/transformer.blocks.6.attn_out.weight.max":4.2508145270403475e-05,"optim/grad/transformer.blocks.13.attn_norm.weight.norm":0.00015370073379017413,"optim/param/transformer.blocks.5.ffn.experts.mlp.w2.min":6.901460514185942e-10,"optim/param/transformer.blocks.15.ff_norm.weight.min":0.5797594785690308,"optim/grad/transformer.blocks.13.ffn.experts.mlp.w2.norm":0.0018643962685018778,"optim/exp_avg/transformer.blocks.4.attn_norm.weight.avg":3.385706008884881e-07,"optim/exp_avg_sq/transformer.blocks.8.ffn.router.layer.weight.norm":4.325331204313443e-08,"optim/grad/transformer.blocks.0.attn_norm.weight.avg":-1.5370898154287715e-07,"optim/exp_avg/transformer.blocks.8.k_norm.weight.max":7.41049643693259e-06,"optim/grad/transformer.blocks.4.q_norm.weight.avg":-1.2212774436193286e-07,"optim/grad/transformer.blocks.3.q_norm.weight.norm":0.00014320763875730336,"optim/grad/transformer.blocks.5.ffn.router.layer.weight.min":1.3642420526593924e-11,"optim/param/transformer.blocks.4.k_norm.weight.norm":45.61902618408203,"optim/grad/transformer.blocks.12.att_proj.weight.avg":-3.040668758447218e-11,"optim/param/transformer.blocks.8.attn_out.weight.norm":44.84370803833008,"optim/exp_avg/transformer.blocks.11.ffn.experts.mlp.w1.norm":0.0005787825211882591,"optim/exp_avg/transformer.blocks.5.ff_norm.weight.avg":-1.0141981476863293e-07,"optim/exp_avg/transformer.blocks.0.q_norm.weight.min":2.8641674101881343e-11,"optim/grad/transformer.blocks.9.att_proj.weight.max":0.00024988147197291255,"optim/exp_avg/transformer.blocks.3.att_proj.weight.norm":0.0027517168782651424,"optim/grad/transformer.blocks.2.q_norm.weight.norm":0.00010433669376652688,"optim/exp_avg_sq/transformer.blocks.4.ffn.experts.mlp.w1.norm":7.780399435830532e-09,"optim/grad/transformer.blocks.10.attn_out.weight.avg":1.3658313369191433e-10,"optim/exp_avg/transformer.blocks.13.att_proj.weight.norm":0.0008100547711364925,"optim/grad/transformer.blocks.5.ff_norm.weight.avg":-3.0708258691447554e-07,"optim/exp_avg/transformer.blocks.3.ffn.router.layer.weight.max":6.487387872766703e-05,"optim/exp_avg/transformer.blocks.4.k_norm.weight.max":1.3057991054665763e-05,"optim/grad/transformer.blocks.12.attn_norm.weight.avg":-1.966057538993482e-07,"optim/param/transformer.blocks.3.ff_norm.weight.avg":0.8857402801513672,"optim/param/transformer.blocks.3.attn_norm.weight.min":0.4252464473247528,"optim/param/transformer.blocks.14.k_norm.weight.norm":41.93832015991211,"optim/grad/transformer.blocks.8.ff_norm.weight.max":3.256750642322004e-05,"optim/exp_avg_sq/transformer.blocks.9.q_norm.weight.norm":3.4245390967413414e-09,"optim/param/transformer.blocks.7.attn_out.weight.avg":4.2207984733977355e-06,"optim/grad/transformer.blocks.0.k_norm.weight.min":2.2089352569309995e-10,"optim/param/transformer.blocks.15.attn_norm.weight.max":0.7211512327194214,"optim/grad/transformer.blocks.2.ffn.router.layer.weight.avg":4.754432403331066e-10,"optim/param/transformer.blocks.12.q_norm.weight.avg":0.9344604015350342,"optim/exp_avg_sq/transformer.blocks.2.ffn.experts.mlp.w1.norm":4.870936542999971e-09,"optim/exp_avg/transformer.blocks.13.ffn.experts.mlp.w2.avg":2.1993235357897767e-11,"optim/grad/transformer.blocks.1.ffn.experts.mlp.w1.max":3.6011839256389067e-05,"optim/param/transformer.blocks.5.att_proj.weight.max":0.18174941837787628,"optim/param/transformer.blocks.11.k_norm.weight.norm":46.351417541503906,"optim/exp_avg/transformer.blocks.3.attn_out.weight.min":2.1579267507013805e-13,"throughput/device/batches_per_second":0.026890330467855387,"optim/param/transformer.blocks.4.attn_norm.weight.avg":0.5840572118759155,"optim/grad/transformer.blocks.8.attn_norm.weight.min":1.23463905765675e-09,"optim/exp_avg/transformer.blocks.3.q_norm.weight.norm":2.025060348387342e-05,"optim/param/transformer.blocks.9.att_proj.weight.avg":-4.77507137475186e-06,"optim/param/transformer.blocks.8.att_proj.weight.norm":78.01382446289062,"optim/grad/transformer.blocks.2.k_norm.weight.avg":-1.9968740616604919e-07,"optim/grad/transformer.blocks.15.q_norm.weight.avg":-5.269351959213964e-07,"optim/exp_avg_sq/transformer.blocks.12.q_norm.weight.min":8.40353245432253e-15,"optim/exp_avg_sq/transformer.blocks.9.k_norm.weight.min":6.744011528018274e-15,"optim/grad/transformer.blocks.8.k_norm.weight.norm":0.00014840032963547856,"optim/param/transformer.blocks.4.ffn.experts.mlp.w1.norm":317.6256408691406,"optim/exp_avg/transformer.blocks.2.q_norm.weight.max":4.7022213038872e-06,"optim/exp_avg/transformer.blocks.5.attn_norm.weight.avg":1.819701367367088e-07,"optim/grad/transformer.blocks.1.attn_out.weight.max":6.737211515428498e-05,"train/MoEZLoss":0.0004764365148730576,"optim/param/transformer.blocks.5.ffn.experts.mlp.w2.avg":-1.2548642189358361e-05,"optim/exp_avg/transformer.blocks.2.attn_norm.weight.min":9.963550817726485e-11,"optim/exp_avg_sq/transformer.blocks.11.ffn.experts.mlp.w1.avg":4.796092654426286e-14,"optim/param/transformer.blocks.11.ff_norm.weight.max":1.2022150754928589,"optim/exp_avg_sq/transformer.blocks.0.q_norm.weight.max":2.2625967766032318e-10,"optim/grad/transformer.blocks.2.ff_norm.weight.min":1.2732925824820995e-10,"optim/grad/transformer.blocks.5.attn_norm.weight.avg":7.907834742582054e-07,"optim/param/transformer.ln_f.weight.min":0.7142730355262756,"optim/param/transformer.blocks.11.attn_norm.weight.avg":0.6369550228118896,"optim/exp_avg/transformer.blocks.5.ffn.router.layer.weight.avg":1.8119783451453486e-10,"optim/grad/transformer.blocks.15.attn_norm.weight.max":3.67700558854267e-05,"optim/exp_avg_sq/transformer.blocks.0.attn_norm.weight.max":1.119045034414512e-08,"optim/exp_avg_sq/transformer.blocks.3.ffn.router.layer.weight.norm":1.3823286337810714e-07,"optim/grad/transformer.blocks.8.q_norm.weight.min":1.149373929365538e-10,"optim/exp_avg_sq/transformer.blocks.5.ffn.router.layer.weight.min":1.0561420661636678e-11,"optim/exp_avg/transformer.blocks.1.ffn.router.layer.weight.avg":7.720249994846995e-10,"optim/param/transformer.blocks.8.ffn.experts.mlp.w2.norm":322.6251220703125,"optim/param/transformer.blocks.11.q_norm.weight.norm":46.75759506225586,"optim/param/transformer.blocks.7.att_proj.weight.norm":65.28775024414062,"optim/param/transformer.blocks.1.ffn.router.layer.weight.norm":5.473254680633545,"optim/param/transformer.blocks.4.att_proj.weight.min":1.7337914437476343e-09,"optim/param/transformer.blocks.7.attn_norm.weight.min":0.5756580829620361,"optim/grad/transformer.blocks.0.ffn.experts.mlp.w2.min":0,"optim/param/transformer.blocks.1.k_norm.weight.min":0.5652487874031067,"optim/param/transformer.blocks.15.ffn.experts.mlp.w2.max":0.5454942584037781,"optim/param/transformer.blocks.0.ffn.router.layer.weight.min":2.436921704429551e-07,"optim/param/transformer.blocks.14.att_proj.weight.norm":78.04842376708984,"optim/grad/transformer.blocks.5.attn_out.weight.min":0,"optim/param/transformer.blocks.8.attn_out.weight.max":0.12561969459056854,"optim/grad/transformer.blocks.15.ffn.experts.mlp.w2.min":0,"optim/exp_avg/transformer.blocks.12.att_proj.weight.min":3.462958150182054e-16,"optim/param/transformer.blocks.12.ffn.experts.mlp.w2.min":1.7988523171030835e-10,"optim/exp_avg_sq/transformer.blocks.4.attn_out.weight.norm":3.4095108958354103e-08,"optim/param/transformer.blocks.9.q_norm.weight.max":1.3977553844451904,"optim/param/transformer.blocks.11.ffn.experts.mlp.w2.avg":-2.0683499315055087e-05,"optim/exp_avg_sq/transformer.blocks.4.attn_out.weight.min":3.677515648774221e-13,"optim/grad/transformer.blocks.7.att_proj.weight.max":0.0005526688764803112,"optim/grad/transformer.blocks.3.attn_out.weight.min":0,"optim/exp_avg_sq/transformer.blocks.9.ffn.router.layer.weight.norm":3.106253387841207e-08,"eval/downstream/mmlu_other_mc_5shot_len_norm":0.2450704276561737,"optim/param/transformer.blocks.7.ffn.experts.mlp.w1.max":0.18143564462661743,"optim/exp_avg_sq/transformer.blocks.7.k_norm.weight.max":8.119255823224591e-10,"optim/param/transformer.blocks.12.attn_norm.weight.avg":0.6846548914909363,"optim/grad/transformer.blocks.3.ffn.experts.mlp.w2.norm":0.00315747014246881,"optim/grad/transformer.blocks.9.ff_norm.weight.min":3.558852768037468e-09,"optim/exp_avg_sq/transformer.blocks.0.ffn.experts.mlp.w2.norm":7.893518616519657e-10,"optim/exp_avg_sq/transformer.blocks.14.attn_norm.weight.min":1.840650028228641e-12,"optim/grad/transformer.blocks.1.attn_norm.weight.norm":0.00040054103010334074,"optim/exp_avg_sq/transformer.blocks.11.ffn.experts.mlp.w2.avg":3.1565841520503346e-14,"optim/param/transformer.blocks.9.k_norm.weight.norm":43.08613204956055,"optim/exp_avg_sq/transformer.blocks.1.ffn.router.layer.weight.max":3.997385178422519e-08,"optim/param/transformer.blocks.15.ffn.router.layer.weight.max":0.16800639033317566,"optim/exp_avg/transformer.blocks.13.ffn.router.layer.weight.norm":0.0006054236437194049,"optim/exp_avg_sq/transformer.blocks.9.ff_norm.weight.max":4.055779601053189e-10,"optim/exp_avg/transformer.blocks.7.attn_out.weight.avg":9.44879244291208e-11,"optim/exp_avg_sq/transformer.blocks.14.attn_norm.weight.max":9.9210206716549e-10,"optim/param/transformer.blocks.15.ff_norm.weight.norm":45.852294921875,"optim/exp_avg/transformer.blocks.5.q_norm.weight.norm":3.909295628545806e-05,"optim/exp_avg/transformer.blocks.1.attn_norm.weight.avg":1.3980931612422864e-07,"optim/exp_avg_sq/transformer.blocks.4.q_norm.weight.min":3.3616268575481934e-15,"optim/exp_avg/transformer.blocks.9.ffn.router.layer.weight.norm":0.0005952771753072739,"optim/exp_avg/transformer.blocks.6.k_norm.weight.max":6.369396942318417e-06,"optim/exp_avg/transformer.blocks.11.k_norm.weight.max":4.037933194922516e-06,"optim/grad/transformer.blocks.11.ffn.experts.mlp.w2.norm":0.002066482789814472,"optim/exp_avg_sq/transformer.blocks.2.ffn.experts.mlp.w2.avg":6.746297585239619e-14,"optim/param/transformer.wte.weight.avg":-4.235075903125107e-05,"optim/exp_avg_sq/transformer.blocks.1.ffn.experts.mlp.w1.avg":4.1834499867098376e-14,"optim/grad/transformer.blocks.11.attn_norm.weight.min":2.4301698431372643e-09,"optim/param/transformer.blocks.3.att_proj.weight.max":0.1774919182062149,"optim/param/transformer.blocks.12.q_norm.weight.norm":42.717445373535156,"optim/exp_avg_sq/transformer.blocks.2.ffn.experts.mlp.w2.max":2.7083462209032128e-11,"optim/exp_avg_sq/transformer.blocks.1.ffn.experts.mlp.w2.max":7.795111778285957e-11,"optim/exp_avg_sq/transformer.blocks.9.q_norm.weight.avg":1.373439730001369e-11,"optim/exp_avg/transformer.blocks.2.ff_norm.weight.avg":-2.1523834448089474e-07,"optim/grad/transformer.blocks.13.attn_out.weight.min":9.094947017729282e-13,"eval/pile-validation/Perplexity":13.576504707336426,"optim/exp_avg_sq/transformer.blocks.2.ffn.experts.mlp.w1.avg":6.238655154543887e-14,"optim/exp_avg/transformer.blocks.0.att_proj.weight.norm":0.001554501592181623,"optim/exp_avg_sq/transformer.blocks.8.k_norm.weight.max":5.734898333997762e-09,"optim/exp_avg_sq/transformer.blocks.14.ffn.experts.mlp.w1.norm":1.5337888736866034e-09,"optim/exp_avg_sq/transformer.blocks.4.att_proj.weight.min":1.7761355520427803e-15,"optim/exp_avg_sq/transformer.blocks.1.ffn.experts.mlp.w2.avg":5.6543284594409715e-14,"optim/exp_avg_sq/transformer.blocks.7.ff_norm.weight.min":2.2135925904775977e-12,"optim/exp_avg/transformer.blocks.12.ffn.experts.mlp.w1.avg":1.167534276723392e-12,"optim/grad/transformer.blocks.8.attn_out.weight.max":3.319288225611672e-05,"optim/grad/transformer.blocks.15.ffn.experts.mlp.w2.avg":6.082272879792683e-11,"optim/grad/transformer.blocks.2.attn_norm.weight.max":0.0004555944469757378,"optim/exp_avg/transformer.blocks.8.ffn.router.layer.weight.avg":1.5518086815546894e-11,"optim/param/transformer.blocks.11.k_norm.weight.min":0.4181261658668518,"optim/param/transformer.blocks.11.att_proj.weight.avg":9.462928574066609e-06,"optim/grad/transformer.blocks.7.ffn.experts.mlp.w1.max":9.323620906798169e-05,"optim/exp_avg_sq/transformer.blocks.15.attn_norm.weight.max":5.189244589587361e-09,"optim/grad/transformer.blocks.8.attn_norm.weight.avg":5.17228180285656e-08,"optim/param/transformer.blocks.2.attn_out.weight.avg":-9.294348274124786e-06,"optim/exp_avg_sq/transformer.blocks.15.ff_norm.weight.norm":4.839760148200867e-09,"optim/exp_avg_sq/transformer.blocks.13.ffn.experts.mlp.w1.min":8.447202826105001e-17,"optim/exp_avg_sq/transformer.blocks.5.ff_norm.weight.max":3.7239624717244624e-09,"optim/grad/transformer.blocks.5.ffn.experts.mlp.w1.max":4.4790736865252256e-05,"optim/param/transformer.blocks.3.ffn.experts.mlp.w1.min":7.765331433429168e-11,"optim/param/transformer.ln_f.weight.norm":56.14033508300781,"optim/grad/transformer.blocks.3.q_norm.weight.min":7.639755494892597e-10,"optim/exp_avg_sq/transformer.blocks.15.attn_norm.weight.min":2.0820890584555274e-12,"optim/param/transformer.blocks.5.attn_norm.weight.min":0.537773609161377,"optim/param/transformer.blocks.6.attn_out.weight.avg":-9.459128705202602e-06,"optim/exp_avg_sq/transformer.blocks.6.attn_out.weight.max":3.089749844864542e-10,"optim/param/transformer.blocks.4.q_norm.weight.norm":46.01416778564453,"optim/exp_avg_sq/transformer.blocks.7.ffn.experts.mlp.w1.avg":6.935237432080521e-14,"optim/param/transformer.blocks.11.ffn.router.layer.weight.max":0.12853318452835083,"optim/grad/transformer.blocks.11.att_proj.weight.min":0,"optim/grad/transformer.blocks.5.attn_norm.weight.min":1.913576852530241e-09,"optim/exp_avg/transformer.wte.weight.norm":0.0008811456500552595,"optim/exp_avg_sq/transformer.blocks.7.ffn.experts.mlp.w2.avg":4.739433942958088e-14,"optim/exp_avg_sq/transformer.blocks.2.k_norm.weight.avg":7.117770782494315e-12,"optim/grad/transformer.blocks.14.att_proj.weight.norm":0.003405991941690445,"optim/exp_avg_sq/transformer.blocks.5.attn_out.weight.norm":1.7661003326452374e-08,"optim/grad/transformer.blocks.5.q_norm.weight.max":3.8726459024474025e-05,"optim/exp_avg/transformer.blocks.10.att_proj.weight.min":3.8750402084168634e-16,"optim/grad/transformer.blocks.13.att_proj.weight.norm":0.0038919621147215366,"optim/grad/transformer.blocks.6.k_norm.weight.avg":-1.0949278816951846e-08,"optim/exp_avg_sq/transformer.blocks.7.ffn.experts.mlp.w2.max":1.8114367644761487e-11,"optim/grad/transformer.blocks.9.attn_out.weight.min":0,"optim/param/transformer.blocks.3.q_norm.weight.norm":43.441165924072266,"optim/exp_avg_sq/transformer.blocks.15.ffn.experts.mlp.w2.avg":1.9042703340837325e-14,"optim/exp_avg_sq/transformer.blocks.7.attn_norm.weight.norm":3.2918105574708534e-08,"optim/exp_avg/transformer.blocks.11.ffn.experts.mlp.w1.min":6.562240057482514e-16,"optim/exp_avg_sq/transformer.blocks.9.attn_out.weight.max":4.674017076156112e-11,"optim/exp_avg/transformer.blocks.11.att_proj.weight.min":1.0649452172428568e-14,"optim/param/transformer.blocks.10.k_norm.weight.min":0.20629028975963593,"optim/param/transformer.blocks.14.q_norm.weight.max":1.3090602159500122,"optim/exp_avg_sq/transformer.blocks.8.attn_norm.weight.norm":1.569664043188368e-08,"optim/param/transformer.blocks.5.ff_norm.weight.norm":43.00033950805664,"optim/grad/transformer.blocks.14.attn_out.weight.norm":0.0020418299827724695,"optim/exp_avg_sq/transformer.blocks.3.ffn.experts.mlp.w1.max":1.8265153833851855e-09,"optim/exp_avg/transformer.blocks.14.att_proj.weight.min":2.9901015203333964e-14,"optim/grad/transformer.blocks.14.ffn.experts.mlp.w2.max":4.722960511571728e-06,"optim/grad/transformer.blocks.0.attn_norm.weight.min":1.5352270565927029e-09,"optim/param/transformer.blocks.15.ff_norm.weight.avg":1.0114269256591797,"optim/grad/transformer.blocks.3.k_norm.weight.avg":2.614494576391735e-07,"optim/exp_avg_sq/transformer.blocks.1.att_proj.weight.max":1.3088057357890648e-06,"optim/exp_avg_sq/transformer.blocks.3.q_norm.weight.norm":2.0818846646619704e-09,"optim/param/transformer.blocks.1.k_norm.weight.max":1.4486390352249146,"optim/exp_avg_sq/transformer.blocks.3.ffn.experts.mlp.w2.min":2.7006101805656995e-15,"optim/exp_avg_sq/transformer.blocks.6.ffn.experts.mlp.w2.max":3.7392111976175535e-12,"optim/exp_avg/transformer.blocks.0.q_norm.weight.avg":1.888022360674313e-08,"optim/exp_avg/transformer.blocks.6.att_proj.weight.max":8.131081267492846e-05,"optim/exp_avg_sq/transformer.blocks.9.ffn.experts.mlp.w2.max":3.681754987688857e-12,"optim/exp_avg_sq/transformer.blocks.6.ffn.experts.mlp.w2.avg":4.406815285406228e-14,"optim/exp_avg_sq/transformer.blocks.9.ffn.experts.mlp.w2.avg":3.587029355186025e-14,"optim/param/transformer.blocks.15.ffn.experts.mlp.w1.min":6.648306349887889e-10,"optim/exp_avg_sq/transformer.blocks.3.q_norm.weight.avg":8.74959826813182e-12,"optim/param/transformer.blocks.9.q_norm.weight.min":0.18859601020812988,"optim/exp_avg_sq/transformer.blocks.9.ffn.experts.mlp.w1.avg":5.388196128424533e-14,"optim/exp_avg_sq/transformer.blocks.6.ffn.experts.mlp.w1.avg":6.42344860570139e-14,"optim/exp_avg_sq/transformer.blocks.12.attn_norm.weight.max":7.84689146993145e-10,"optim/grad/transformer.blocks.9.ff_norm.weight.max":2.2854350390844047e-05,"optim/exp_avg/transformer.blocks.12.ffn.experts.mlp.w1.max":1.7828873524194933e-06,"optim/param/transformer.blocks.3.ffn.router.layer.weight.norm":6.566807270050049,"optim/exp_avg/transformer.blocks.12.ff_norm.weight.avg":-1.8086819864038262e-07,"optim/param/transformer.blocks.12.ffn.experts.mlp.w1.norm":331.1051025390625,"optim/grad/transformer.blocks.9.ffn.router.layer.weight.max":0.00015625273226760328,"optim/param/transformer.blocks.10.k_norm.weight.norm":43.82011413574219,"optim/param/transformer.blocks.1.attn_norm.weight.min":0.30649274587631226,"optim/exp_avg_sq/transformer.blocks.12.attn_norm.weight.min":2.7102335133089017e-12,"optim/param/transformer.blocks.0.ffn.router.layer.weight.avg":-0.0001461859792470932,"optim/param/transformer.blocks.10.attn_norm.weight.norm":33.01304626464844,"optim/exp_avg/transformer.blocks.2.k_norm.weight.max":5.3162648327997886e-06,"optim/grad/transformer.blocks.14.q_norm.weight.norm":7.216040103230625e-05,"optim/grad/transformer.blocks.7.k_norm.weight.avg":-1.9662306272039132e-07,"optim/param/transformer.blocks.6.att_proj.weight.min":2.3533421877885985e-09,"optim/grad/transformer.blocks.9.ff_norm.weight.norm":9.534216223983094e-05,"optim/exp_avg/transformer.blocks.9.ff_norm.weight.min":2.4203847259762767e-10,"optim/param/transformer.blocks.4.attn_norm.weight.max":0.6697368025779724,"optim/grad/transformer.blocks.11.attn_out.weight.avg":8.294313014189925e-10,"throughput/total_training_Gflops":5.0714543978815106e+11,"optim/param/transformer.blocks.2.att_proj.weight.min":2.200297499754811e-09,"optim/param/transformer.blocks.1.k_norm.weight.avg":0.8884304165840149,"optim/exp_avg/transformer.blocks.3.ffn.router.layer.weight.min":8.708499416379412e-13,"optim/grad/transformer.blocks.9.att_proj.weight.min":0,"optim/param/transformer.blocks.11.attn_norm.weight.norm":28.833511352539062,"optim/exp_avg_sq/transformer.blocks.12.ffn.experts.mlp.w1.min":3.1311224051833414e-16,"optim/exp_avg_sq/transformer.blocks.2.ffn.router.layer.weight.min":5.334091675301966e-12,"optim/exp_avg_sq/transformer.blocks.11.att_proj.weight.avg":2.049630430656091e-12,"optim/exp_avg/transformer.blocks.13.ffn.experts.mlp.w1.min":7.831675100642733e-16,"optim/exp_avg/transformer.blocks.9.attn_norm.weight.min":4.0942790824338715e-10,"optim/exp_avg_sq/transformer.blocks.4.attn_norm.weight.min":8.894897839117899e-12,"optim/exp_avg/transformer.blocks.10.ffn.router.layer.weight.norm":0.0005636758869513869,"optim/exp_avg_sq/transformer.blocks.5.att_proj.weight.min":2.3308667465620414e-15,"optim/grad/transformer.blocks.15.attn_norm.weight.avg":-1.9050780508678145e-07,"optim/param/transformer.blocks.6.attn_norm.weight.norm":28.34187126159668,"optim/grad/transformer.blocks.5.ffn.experts.mlp.w1.norm":0.003440440632402897,"optim/param/transformer.blocks.7.ffn.experts.mlp.w2.avg":-2.649943417054601e-05,"optim/exp_avg/transformer.blocks.7.att_proj.weight.norm":0.0019303483422845602,"optim/param/transformer.blocks.15.att_proj.weight.avg":8.23014670459088e-06,"optim/param/transformer.blocks.12.ffn.router.layer.weight.avg":0.00024864557781256735,"optim/exp_avg/transformer.blocks.5.ffn.experts.mlp.w1.min":8.939689486857924e-16,"optim/exp_avg/transformer.blocks.7.k_norm.weight.min":2.1293404539601823e-11,"optim/grad/transformer.blocks.4.attn_out.weight.norm":0.005833115428686142,"optim/grad/transformer.blocks.9.ff_norm.weight.avg":4.164852640542449e-08,"optim/grad/transformer.blocks.2.ffn.experts.mlp.w1.norm":0.0029189386405050755,"optim/exp_avg/transformer.blocks.12.ff_norm.weight.norm":1.9400098608457483e-05,"optim/exp_avg/transformer.blocks.9.attn_out.weight.min":7.111890557801731e-14,"optim/exp_avg/transformer.blocks.0.k_norm.weight.norm":1.67554371728329e-05,"optim/exp_avg/transformer.blocks.11.ffn.router.layer.weight.min":2.479121422038766e-11,"optim/exp_avg/transformer.blocks.14.ffn.router.layer.weight.norm":0.0005573980743065476,"optim/param/transformer.blocks.2.attn_out.weight.norm":34.78773880004883,"optim/exp_avg_sq/transformer.ff_out.weight.min":4.022804705872326e-23,"optim/exp_avg_sq/transformer.blocks.0.ffn.experts.mlp.w2.min":2.3218729838153197e-17,"optim/exp_avg/transformer.blocks.4.ffn.experts.mlp.w2.avg":-5.0144024349441096e-11,"optim/param/transformer.blocks.15.k_norm.weight.avg":0.8930294513702393,"optim/exp_avg/transformer.blocks.9.ffn.experts.mlp.w2.norm":0.0005006617866456509,"optim/grad/transformer.blocks.4.ffn.router.layer.weight.max":0.0002754360029939562,"optim/grad/transformer.blocks.10.ffn.router.layer.weight.avg":-8.405809381883955e-11,"optim/param/transformer.blocks.7.ffn.experts.mlp.w2.min":6.43128675315019e-11,"optim/param/transformer.blocks.10.att_proj.weight.norm":79.6659927368164,"optim/exp_avg_sq/transformer.blocks.0.ffn.experts.mlp.w1.max":5.036755457155095e-10,"optim/param/transformer.blocks.1.ff_norm.weight.avg":0.8182798624038696,"optim/param/transformer.blocks.12.att_proj.weight.norm":78.70342254638672,"optim/param/transformer.blocks.7.attn_norm.weight.norm":31.06488037109375,"optim/param/transformer.blocks.13.ffn.router.layer.weight.min":1.34944841079232e-07,"optim/exp_avg/transformer.blocks.11.ffn.router.layer.weight.max":2.5678071324364282e-05,"optim/exp_avg/transformer.blocks.7.att_proj.weight.max":8.057623927015811e-05,"optim/grad/transformer.blocks.1.ffn.router.layer.weight.norm":0.0026266202330589294,"optim/exp_avg_sq/transformer.blocks.5.attn_out.weight.min":6.373559449310284e-13,"optim/exp_avg/transformer.blocks.5.ff_norm.weight.max":6.8728277256013826e-06,"optim/exp_avg_sq/transformer.blocks.7.attn_out.weight.max":7.069925894720441e-11,"optim/param/transformer.blocks.12.attn_norm.weight.max":0.7511046528816223,"optim/exp_avg_sq/transformer.blocks.2.attn_norm.weight.max":1.1489136397813127e-07,"optim/param/transformer.blocks.6.attn_norm.weight.min":0.5412388443946838,"optim/exp_avg_sq/transformer.blocks.11.ff_norm.weight.avg":3.411155689511669e-12,"optim/exp_avg_sq/transformer.blocks.11.k_norm.weight.min":6.322306010832354e-15,"optim/param/transformer.blocks.9.attn_norm.weight.min":0.5692965388298035,"optim/exp_avg_sq/transformer.blocks.11.attn_norm.weight.max":1.3796466191706713e-09,"optim/param/transformer.blocks.2.q_norm.weight.norm":43.067283630371094,"optim/grad/transformer.blocks.15.ffn.experts.mlp.w1.max":3.692430618684739e-05,"optim/exp_avg/transformer.blocks.15.attn_out.weight.avg":1.705251911898742e-10,"optim/grad/transformer.blocks.12.attn_norm.weight.max":3.06632173305843e-05,"optim/param/transformer.blocks.3.ff_norm.weight.norm":40.14705276489258,"optim/param/transformer.blocks.12.k_norm.weight.min":0.42344802618026733,"optim/exp_avg_sq/transformer.blocks.7.q_norm.weight.norm":2.383981234643784e-09,"optim/exp_avg/transformer.blocks.15.q_norm.weight.norm":1.0116453267983161e-05,"optim/param/transformer.blocks.10.q_norm.weight.max":1.3735781908035278,"optim/param/transformer.blocks.0.attn_out.weight.min":4.762047645101575e-09,"optim/grad/transformer.blocks.2.k_norm.weight.norm":0.0001086815245798789,"optim/exp_avg_sq/transformer.blocks.4.ffn.router.layer.weight.min":6.121260616442914e-12,"optim/exp_avg_sq/transformer.blocks.11.attn_norm.weight.min":4.702642355386422e-12,"optim/exp_avg_sq/transformer.blocks.14.att_proj.weight.norm":3.760823474863173e-08,"optim/grad/transformer.blocks.0.attn_norm.weight.norm":0.00027520774165168405,"optim/exp_avg/transformer.blocks.1.ffn.router.layer.weight.norm":0.0005856141215190291,"optim/exp_avg/transformer.blocks.15.ffn.experts.mlp.w2.max":1.0660080533853034e-06,"optim/grad/transformer.blocks.10.k_norm.weight.norm":0.00012696707563009113,"optim/exp_avg_sq/transformer.blocks.14.attn_out.weight.avg":1.0006203764872934e-12,"optim/exp_avg_sq/transformer.blocks.12.ff_norm.weight.max":2.130844251047037e-10,"eval/downstream/mmlu_humanities_mc_5shot_test_len_norm":0.2548352777957916,"optim/exp_avg/transformer.blocks.2.ffn.router.layer.weight.norm":0.0006043441244401038,"optim/exp_avg_sq/transformer.blocks.4.ffn.router.layer.weight.avg":6.893541293351291e-11,"optim/param/transformer.blocks.5.att_proj.weight.norm":78.50486755371094,"optim/exp_avg_sq/transformer.blocks.9.attn_out.weight.avg":3.068956764065711e-12,"optim/param/transformer.ff_out.weight.norm":330.1094970703125,"optim/exp_avg_sq/transformer.blocks.2.attn_norm.weight.norm":1.457573119978406e-07,"optim/grad/transformer.blocks.7.ffn.router.layer.weight.min":9.094947017729282e-11,"optim/exp_avg_sq/transformer.blocks.13.ffn.experts.mlp.w2.min":1.8053335982978037e-17,"optim/exp_avg/transformer.blocks.1.ff_norm.weight.avg":-1.2839649343732162e-07,"optim/grad/transformer.blocks.8.ffn.router.layer.weight.min":4.3655745685100555e-11,"optim/grad/transformer.blocks.8.ffn.experts.mlp.w1.avg":5.0247594279850816e-11,"optim/grad/transformer.blocks.7.ffn.experts.mlp.w2.max":4.537291715678293e-06,"eval/downstream/mmlu_stem_mc_5shot_test_len_norm":0.24519549310207367,"optim/param/transformer.blocks.2.ffn.experts.mlp.w2.norm":302.1220397949219,"optim/exp_avg/transformer.blocks.14.attn_out.weight.avg":-6.87402287868899e-11,"optim/grad/transformer.blocks.5.ffn.experts.mlp.w2.max":5.214315933699254e-06,"optim/exp_avg_sq/transformer.blocks.10.att_proj.weight.avg":3.058875852265941e-12,"optim/exp_avg_sq/transformer.blocks.6.attn_out.weight.avg":6.415851357133295e-12,"optim/exp_avg_sq/transformer.blocks.1.ffn.experts.mlp.w2.norm":2.3190582787435687e-09,"optim/grad/transformer.blocks.7.ffn.experts.mlp.w1.norm":0.0030857969541102648,"optim/exp_avg_sq/transformer.blocks.12.att_proj.weight.avg":1.5017674603878817e-12,"optim/param/transformer.blocks.13.k_norm.weight.norm":42.27415084838867,"optim/exp_avg/transformer.blocks.8.attn_norm.weight.avg":2.3608433252775285e-07,"optim/grad/transformer.blocks.11.attn_norm.weight.norm":0.00019124925893265754,"optim/exp_avg_sq/transformer.blocks.7.ff_norm.weight.avg":6.31357336203231e-12,"optim/param/transformer.blocks.5.ffn.experts.mlp.w1.min":5.064702476853533e-11,"optim/grad/transformer.blocks.12.ffn.router.layer.weight.avg":-9.841372161645268e-11,"optim/exp_avg/transformer.blocks.13.q_norm.weight.avg":-8.719658239897399e-08,"optim/grad/transformer.blocks.1.attn_out.weight.min":0,"optim/exp_avg_sq/transformer.blocks.4.k_norm.weight.min":3.78817994429125e-15,"optim/exp_avg_sq/transformer.blocks.10.ffn.experts.mlp.w1.norm":8.042099763905242e-10,"optim/param/transformer.blocks.7.q_norm.weight.avg":0.8876177072525024,"optim/exp_avg_sq/transformer.blocks.3.ffn.experts.mlp.w1.min":2.362019400752193e-16,"optim/exp_avg_sq/transformer.blocks.6.q_norm.weight.max":3.0744953249950413e-09,"optim/grad/transformer.blocks.3.attn_out.weight.norm":0.005557215306907892,"optim/exp_avg_sq/transformer.blocks.14.att_proj.weight.max":2.8003153129674274e-09,"optim/exp_avg_sq/transformer.blocks.15.ffn.experts.mlp.w2.norm":3.8806852176165307e-10,"optim/param/transformer.ff_out.weight.avg":-6.848132761660963e-05,"optim/exp_avg_sq/transformer.blocks.9.attn_norm.weight.max":6.911679673038407e-09,"optim/exp_avg/transformer.blocks.9.att_proj.weight.max":3.101992842857726e-05,"optim/grad/transformer.blocks.6.q_norm.weight.norm":0.0002516362874303013,"optim/grad/transformer.blocks.13.ffn.router.layer.weight.min":3.2741809263825417e-11,"optim/exp_avg_sq/transformer.blocks.14.k_norm.weight.max":1.3698253642502323e-10,"optim/param/transformer.blocks.7.ffn.experts.mlp.w1.avg":5.052676715422422e-05,"optim/grad/transformer.blocks.0.ff_norm.weight.max":2.6426278054714203e-05,"optim/exp_avg_sq/transformer.blocks.3.ff_norm.weight.max":2.7921609468961606e-09,"optim/exp_avg/transformer.blocks.9.ffn.router.layer.weight.min":6.031739344103393e-12,"optim/exp_avg/transformer.blocks.12.q_norm.weight.max":4.6169484448910225e-06,"optim/exp_avg/transformer.blocks.13.ffn.router.layer.weight.max":3.648741403594613e-05,"optim/grad/transformer.blocks.7.ff_norm.weight.avg":1.0989707277531124e-07,"optim/exp_avg_sq/transformer.blocks.2.ffn.router.layer.weight.avg":5.621090398699735e-11,"optim/exp_avg_sq/transformer.blocks.15.ffn.router.layer.weight.norm":5.6146731708395237e-08,"optim/param/transformer.blocks.11.ffn.experts.mlp.w1.avg":2.0830075300182216e-05,"optim/exp_avg_sq/transformer.blocks.11.ffn.experts.mlp.w2.norm":4.397357478147512e-10,"optim/param/transformer.blocks.1.ffn.experts.mlp.w2.norm":295.5284423828125,"train/Perplexity":3.581298215424414,"optim/exp_avg/transformer.blocks.7.attn_out.weight.min":4.838650300201773e-14,"optim/grad/transformer.wte.weight.min":0,"optim/param/transformer.blocks.8.ffn.router.layer.weight.norm":7.536323547363281,"optim/exp_avg/transformer.blocks.13.ffn.router.layer.weight.min":3.786573919001279e-12,"optim/exp_avg/transformer.blocks.12.k_norm.weight.min":1.9974316645132717e-10,"optim/exp_avg_sq/transformer.blocks.12.ffn.experts.mlp.w2.norm":3.77885278624035e-10,"optim/param/transformer.blocks.7.ffn.router.layer.weight.avg":0.00042000520625151694,"optim/exp_avg/transformer.blocks.6.attn_norm.weight.avg":1.9121328875826293e-07,"optim/exp_avg_sq/transformer.blocks.1.att_proj.weight.norm":5.511218205356272e-06,"optim/grad/transformer.blocks.6.ffn.router.layer.weight.norm":0.002850621473044157,"optim/exp_avg/transformer.blocks.11.attn_out.weight.avg":-1.0597357713582056e-10,"optim/grad/transformer.blocks.8.ffn.experts.mlp.w2.avg":1.596543383608484e-11,"optim/exp_avg_sq/transformer.blocks.6.ff_norm.weight.norm":4.414641097127969e-09,"optim/grad/transformer.blocks.6.attn_out.weight.min":9.094947017729282e-13,"eval/pile-validation/CrossEntropyLoss":2.6083407402038574,"optim/exp_avg_sq/transformer.wte.weight.norm":6.943162134120939e-08,"optim/grad/transformer.blocks.11.att_proj.weight.max":0.00010383529297541827,"optim/exp_avg/transformer.blocks.4.ff_norm.weight.avg":-1.4488840349713428e-07,"optim/exp_avg/transformer.blocks.8.q_norm.weight.avg":-1.0813476336579697e-07,"optim/exp_avg_sq/transformer.blocks.4.att_proj.weight.avg":2.583865933714069e-11,"optim/exp_avg/transformer.blocks.14.ff_norm.weight.min":9.192157590653949e-10,"optim/exp_avg_sq/transformer.blocks.10.ffn.experts.mlp.w1.avg":4.99794229982288e-14,"optim/exp_avg_sq/transformer.blocks.14.ffn.experts.mlp.w1.max":1.9664152250964406e-10,"optim/exp_avg/transformer.blocks.10.attn_out.weight.avg":-1.1135335709067107e-10,"optim/exp_avg_sq/transformer.blocks.3.attn_out.weight.norm":2.814734756384496e-08,"optim/exp_avg/transformer.blocks.0.k_norm.weight.max":2.923316287706257e-06,"optim/param/transformer.blocks.0.k_norm.weight.avg":0.8089560270309448,"optim/exp_avg/transformer.blocks.11.ffn.experts.mlp.w2.max":5.309162247613131e-07,"optim/grad/transformer.blocks.11.ff_norm.weight.min":1.204170985147357e-09,"optim/exp_avg/transformer.blocks.9.attn_norm.weight.norm":4.590921525959857e-05,"optim/exp_avg/transformer.blocks.4.ffn.router.layer.weight.max":4.794107371708378e-05,"optim/exp_avg_sq/transformer.blocks.13.ffn.experts.mlp.w2.max":2.8745302579208376e-12,"optim/param/transformer.blocks.14.ff_norm.weight.norm":49.93574142456055,"optim/grad/transformer.blocks.8.k_norm.weight.max":2.8122536605224013e-05,"optim/param/transformer.blocks.14.ff_norm.weight.min":0.627011239528656,"optim/param/transformer.blocks.2.attn_norm.weight.min":0.39126139879226685,"optim/exp_avg_sq/transformer.blocks.12.q_norm.weight.avg":5.54101816513386e-12,"optim/exp_avg_sq/transformer.blocks.0.k_norm.weight.max":9.850619625551005e-11,"optim/exp_avg_sq/transformer.blocks.2.ff_norm.weight.norm":4.477892723286914e-09,"optim/grad/transformer.blocks.15.attn_out.weight.min":0,"optim/param/transformer.blocks.11.attn_out.weight.avg":2.00164649868384e-05,"optim/param/transformer.blocks.6.attn_out.weight.norm":42.88064193725586,"optim/exp_avg_sq/transformer.blocks.10.ffn.router.layer.weight.min":6.966998158941529e-12,"optim/exp_avg/transformer.blocks.2.att_proj.weight.max":0.00015814648941159248,"optim/exp_avg/transformer.blocks.3.ffn.experts.mlp.w2.min":1.5279648511344005e-16,"optim/grad/transformer.blocks.8.attn_out.weight.min":9.094947017729282e-13,"optim/exp_avg/transformer.blocks.7.ffn.router.layer.weight.avg":1.1288303625178742e-11,"eval/dolma_common-crawl-validation/CrossEntropyLoss":3.7682061195373535,"optim/exp_avg_sq/transformer.blocks.1.k_norm.weight.avg":4.993421474919213e-12,"optim/grad/transformer.blocks.8.q_norm.weight.avg":-3.376045469849487e-07,"optim/param/transformer.wte.weight.norm":270.8904113769531,"optim/grad/transformer.blocks.13.ff_norm.weight.min":7.60337570682168e-10,"optim/grad/transformer.blocks.15.ffn.experts.mlp.w1.min":0,"optim/grad/transformer.blocks.14.ffn.experts.mlp.w1.norm":0.002369348658248782,"optim/exp_avg/transformer.blocks.3.attn_out.weight.avg":1.812223981989547e-10,"optim/exp_avg/transformer.blocks.12.ffn.experts.mlp.w1.norm":0.0005523672443814576,"optim/exp_avg_sq/transformer.blocks.10.ffn.experts.mlp.w2.avg":3.3560251745581166e-14,"optim/exp_avg/transformer.blocks.10.ffn.experts.mlp.w1.avg":-4.796386812028208e-13,"optim/param/transformer.blocks.0.attn_norm.weight.max":0.7420278191566467,"optim/param/transformer.blocks.9.attn_out.weight.norm":46.943485260009766,"optim/exp_avg/transformer.blocks.13.ff_norm.weight.avg":-1.521969750228891e-07,"optim/exp_avg/transformer.blocks.0.ffn.experts.mlp.w2.max":2.1789951460959855e-06,"optim/param/transformer.blocks.14.ffn.router.layer.weight.max":0.1071082055568695,"optim/exp_avg/transformer.blocks.6.ffn.experts.mlp.w1.min":1.3274996810900934e-16,"optim/grad/transformer.blocks.2.k_norm.weight.min":2.227693585155066e-10,"optim/param/transformer.blocks.11.ffn.experts.mlp.w1.min":3.4724420561182967e-10,"optim/exp_avg/transformer.blocks.0.ffn.experts.mlp.w1.max":6.196706635819282e-06,"optim/exp_avg/transformer.blocks.13.ffn.experts.mlp.w2.norm":0.00042352755554020405,"optim/exp_avg/transformer.blocks.6.k_norm.weight.norm":2.855637649190612e-05,"optim/grad/transformer.blocks.0.k_norm.weight.avg":-2.487490746716503e-07,"optim/grad/transformer.blocks.4.k_norm.weight.max":6.338536331895739e-05,"optim/exp_avg/transformer.blocks.9.ffn.experts.mlp.w2.min":3.252477344932057e-16,"optim/param/transformer.blocks.3.ff_norm.weight.max":1.0324276685714722,"optim/exp_avg/transformer.blocks.2.att_proj.weight.norm":0.002940640551969409,"optim/exp_avg/transformer.blocks.10.k_norm.weight.avg":-1.7705170307635854e-07,"optim/param/transformer.blocks.12.q_norm.weight.max":1.3309153318405151,"optim/exp_avg_sq/transformer.blocks.5.attn_out.weight.avg":7.134538185932238e-12,"optim/param/transformer.blocks.15.attn_out.weight.norm":37.62968826293945,"optim/grad/transformer.blocks.4.ffn.router.layer.weight.min":6.912159733474255e-11,"optim/param/transformer.blocks.7.ffn.experts.mlp.w2.max":0.2703573703765869,"optim/exp_avg_sq/transformer.blocks.10.ff_norm.weight.max":1.4328559172493982e-10,"optim/grad/transformer.blocks.11.q_norm.weight.avg":9.445043502864792e-08,"optim/param/transformer.blocks.5.ffn.experts.mlp.w2.norm":315.8514404296875,"optim/param/transformer.blocks.11.att_proj.weight.min":4.914193940663836e-09,"optim/exp_avg/transformer.ff_out.weight.max":0.00017340373597107828,"optim/grad/transformer.blocks.13.ff_norm.weight.norm":8.286332013085485e-05,"optim/exp_avg/transformer.blocks.2.ffn.experts.mlp.w1.min":5.201355102171985e-16,"optim/grad/transformer.blocks.4.q_norm.weight.min":8.469669410260394e-11,"optim/grad/transformer.blocks.6.k_norm.weight.norm":0.0002392242749920115,"optim/grad/transformer.blocks.5.attn_out.weight.max":2.4131863028742373e-05,"optim/exp_avg_sq/transformer.blocks.1.q_norm.weight.avg":4.665534451830933e-12,"optim/exp_avg_sq/transformer.blocks.4.att_proj.weight.norm":3.4487636639823904e-06,"optim/grad/transformer.blocks.3.attn_out.weight.max":7.598836964461952e-05,"optim/grad/transformer.blocks.9.ffn.experts.mlp.w2.norm":0.002207997487857938,"optim/exp_avg_sq/transformer.blocks.9.ffn.experts.mlp.w1.norm":1.0025609231689714e-09,"optim/param/transformer.blocks.8.ff_norm.weight.min":0.606308102607727,"optim/grad/transformer.blocks.2.att_proj.weight.avg":3.8900369037087046e-10,"optim/exp_avg_sq/transformer.blocks.6.ffn.router.layer.weight.max":1.4903626777140744e-08,"optim/exp_avg_sq/transformer.blocks.5.q_norm.weight.avg":7.12107040001797e-11,"optim/param/transformer.blocks.1.attn_out.weight.avg":2.9805078156641684e-06,"optim/exp_avg_sq/transformer.blocks.2.attn_out.weight.max":1.3961398703798977e-09,"optim/exp_avg/transformer.blocks.2.attn_out.weight.norm":0.0011015452910214663,"optim/exp_avg_sq/transformer.blocks.2.ffn.router.layer.weight.norm":1.1677200717485903e-07,"optim/exp_avg_sq/transformer.blocks.15.q_norm.weight.norm":5.301577510330446e-10,"optim/param/transformer.blocks.14.attn_norm.weight.max":0.7104662656784058,"optim/grad/transformer.blocks.14.ffn.experts.mlp.w1.max":2.267270247102715e-05,"optim/param/transformer.blocks.15.q_norm.weight.max":1.2226364612579346,"optim/grad/transformer.blocks.14.attn_out.weight.avg":2.1974427832915921e-10,"optim/exp_avg_sq/transformer.blocks.10.att_proj.weight.min":1.0659946698886812e-15,"optim/grad/transformer.blocks.10.ffn.experts.mlp.w1.avg":-1.7008790209604996e-11,"optim/exp_avg_sq/transformer.blocks.1.ffn.router.layer.weight.norm":1.5286693155758257e-07,"optim/exp_avg_sq/transformer.blocks.12.att_proj.weight.min":2.7687980268855652e-15,"optim/exp_avg_sq/transformer.blocks.12.ffn.experts.mlp.w1.norm":7.308447735887569e-10,"optim/exp_avg_sq/transformer.blocks.7.att_proj.weight.avg":9.963418111380573e-12,"optim/exp_avg_sq/transformer.blocks.4.ffn.experts.mlp.w2.avg":6.135237852695e-14,"optim/exp_avg_sq/transformer.blocks.3.k_norm.weight.max":1.5992372981443737e-09,"optim/exp_avg/transformer.blocks.4.att_proj.weight.max":0.00013412555563263595,"optim/exp_avg_sq/transformer.blocks.5.ffn.experts.mlp.w1.avg":8.73701474666741e-14,"optim/param/transformer.blocks.5.q_norm.weight.avg":0.9934782981872559,"optim/grad/transformer.blocks.9.ffn.experts.mlp.w2.min":0,"optim/grad/transformer.blocks.1.att_proj.weight.max":0.00216025672852993,"optim/exp_avg_sq/transformer.blocks.10.ffn.router.layer.weight.norm":2.407275445648338e-08,"optim/param/transformer.blocks.13.k_norm.weight.avg":0.924190878868103,"optim/exp_avg_sq/transformer.blocks.13.ff_norm.weight.max":2.0571805370295237e-10,"optim/exp_avg_sq/transformer.blocks.4.ffn.experts.mlp.w2.max":1.226344546745306e-11,"optim/exp_avg/transformer.blocks.11.ffn.experts.mlp.w2.min":1.345480072977934e-17,"optim/exp_avg/transformer.blocks.8.q_norm.weight.min":7.074081598279491e-11,"optim/exp_avg/transformer.blocks.10.ff_norm.weight.avg":-1.267081302103179e-07,"optim/exp_avg_sq/transformer.blocks.5.ffn.experts.mlp.w2.avg":5.6092947669540705e-14,"optim/exp_avg/transformer.blocks.6.ffn.router.layer.weight.max":4.226964301778935e-05,"optim/exp_avg/transformer.blocks.15.ffn.experts.mlp.w2.avg":-2.352978055453203e-12,"optim/grad/transformer.blocks.1.ffn.experts.mlp.w2.max":1.787462315405719e-05,"optim/exp_avg_sq/transformer.blocks.4.ffn.experts.mlp.w1.avg":8.743902818594482e-14,"optim/exp_avg_sq/transformer.blocks.11.attn_norm.weight.avg":1.619214484716025e-11,"optim/exp_avg_sq/transformer.blocks.2.q_norm.weight.norm":1.145545769354328e-09,"optim/exp_avg_sq/transformer.blocks.5.ffn.experts.mlp.w2.max":1.1308343150773226e-11,"optim/grad/transformer.blocks.15.ff_norm.weight.max":4.874751903116703e-05,"optim/exp_avg/transformer.blocks.11.q_norm.weight.min":7.354589159902503e-11,"optim/grad/transformer.blocks.1.ffn.experts.mlp.w1.norm":0.0024062693119049072,"optim/grad/transformer.blocks.7.att_proj.weight.norm":0.011064941994845867,"optim/param/transformer.blocks.3.ffn.experts.mlp.w2.min":1.1677374345264724e-10,"optim/param/transformer.blocks.5.q_norm.weight.norm":45.222110748291016,"optim/exp_avg_sq/transformer.blocks.11.ffn.experts.mlp.w1.norm":8.179180666090247e-10,"optim/param/transformer.blocks.15.q_norm.weight.norm":40.905731201171875,"optim/exp_avg/transformer.blocks.7.k_norm.weight.norm":3.776157245738432e-05,"optim/exp_avg/transformer.blocks.10.ffn.experts.mlp.w1.max":1.4113251154412865e-06,"optim/grad/transformer.blocks.2.q_norm.weight.min":1.0868461686186492e-10,"optim/grad/transformer.blocks.4.ffn.experts.mlp.w1.min":0,"optim/grad/transformer.blocks.6.att_proj.weight.max":0.0006644268869422376,"eval/downstream/sciq_acc":0.8119999766349792,"optim/exp_avg_sq/transformer.blocks.0.ff_norm.weight.avg":2.1435533469754686e-12,"optim/param/transformer.blocks.2.ffn.router.layer.weight.max":0.13738596439361572,"optim/grad/transformer.blocks.9.k_norm.weight.min":2.8819613362429664e-10,"optim/param/transformer.blocks.9.ffn.router.layer.weight.norm":7.518118381500244,"optim/exp_avg_sq/transformer.blocks.10.q_norm.weight.min":3.931458108418157e-15,"optim/exp_avg_sq/transformer.ln_f.weight.max":1.8374466392856448e-08,"optim/param/transformer.blocks.13.ffn.router.layer.weight.norm":7.546463489532471,"optim/param/transformer.blocks.5.att_proj.weight.min":2.6410074127625194e-09,"optim/exp_avg/transformer.blocks.7.attn_out.weight.norm":0.0008706492371857166,"optim/exp_avg_sq/transformer.blocks.15.ffn.experts.mlp.w1.norm":5.016947746128153e-09,"optim/exp_avg/transformer.blocks.14.attn_norm.weight.avg":-1.1384733156205584e-09,"optim/exp_avg/transformer.blocks.15.att_proj.weight.norm":0.0006941017927601933,"optim/grad/transformer.blocks.11.ffn.router.layer.weight.norm":0.0025049117393791676,"optim/grad/transformer.blocks.9.attn_out.weight.max":1.6267205865005963e-05,"optim/grad/transformer.blocks.9.q_norm.weight.min":8.822098607197404e-11,"optim/exp_avg_sq/transformer.blocks.10.ffn.experts.mlp.w2.norm":4.5867040721070396e-10,"optim/param/transformer.blocks.3.ffn.experts.mlp.w2.avg":-5.348987997422228e-06,"optim/grad/transformer.ff_out.weight.norm":0.018048468977212906,"optim/grad/transformer.blocks.10.attn_norm.weight.norm":0.00019374473777133971,"optim/exp_avg_sq/transformer.blocks.7.q_norm.weight.max":1.0170371211870588e-09,"optim/param/transformer.blocks.11.ffn.experts.mlp.w2.max":0.4271271228790283,"optim/exp_avg_sq/transformer.blocks.11.ffn.router.layer.weight.avg":4.8217797810057306e-11,"optim/grad/transformer.blocks.0.q_norm.weight.min":1.1577867553569376e-09,"optim/grad/transformer.blocks.5.ff_norm.weight.norm":0.00017078516248147935,"optim/exp_avg/transformer.blocks.14.k_norm.weight.min":2.589229941563076e-10,"optim/param/transformer.blocks.12.ffn.experts.mlp.w2.norm":333.7305603027344,"optim/exp_avg_sq/transformer.blocks.8.q_norm.weight.min":2.3671142509903683e-15,"optim/grad/transformer.blocks.0.att_proj.weight.avg":8.972800280560023e-10,"optim/grad/transformer.blocks.0.k_norm.weight.norm":7.767367787891999e-05,"optim/exp_avg/transformer.blocks.4.ffn.experts.mlp.w2.norm":0.0006451293011195958,"eval/downstream/arc_easy_acc":0.4140350818634033,"optim/param/transformer.blocks.15.ffn.experts.mlp.w2.avg":7.953858585096896e-06,"optim/exp_avg/transformer.blocks.10.ffn.experts.mlp.w2.norm":0.00048535503447055817,"optim/exp_avg_sq/transformer.blocks.9.ffn.router.layer.weight.avg":5.211282794181926e-11,"optim/grad/transformer.blocks.1.ffn.router.layer.weight.min":4.9112713895738125e-11,"optim/param/transformer.blocks.3.k_norm.weight.norm":43.29731750488281,"optim/grad/transformer.ff_out.weight.avg":-9.85105118138374e-13,"optim/grad/transformer.blocks.7.ff_norm.weight.norm":0.00011116664973087609,"optim/grad/transformer.blocks.0.ff_norm.weight.min":1.91562321560923e-10,"optim/exp_avg/transformer.blocks.8.k_norm.weight.norm":3.090907557634637e-05,"optim/param/transformer.blocks.13.ffn.router.layer.weight.max":0.124192014336586,"optim/param/transformer.blocks.15.ffn.router.layer.weight.norm":8.752854347229004,"optim/param/transformer.blocks.7.ffn.router.layer.weight.min":2.0641762148443377e-06,"optim/exp_avg_sq/transformer.blocks.10.ff_norm.weight.norm":2.8439497978816064e-10,"optim/exp_avg_sq/transformer.blocks.13.att_proj.weight.max":1.5963123045636962e-09,"optim/exp_avg/transformer.blocks.13.attn_norm.weight.norm":2.8688828024314716e-05,"optim/exp_avg_sq/transformer.blocks.6.att_proj.weight.norm":1.6392179986723932e-06,"optim/exp_avg/transformer.blocks.4.ffn.experts.mlp.w2.max":1.6112534240164678e-06,"optim/exp_avg_sq/transformer.blocks.8.ff_norm.weight.max":1.392303383696003e-09,"optim/exp_avg/transformer.blocks.4.ffn.experts.mlp.w1.max":7.74352974985959e-06,"optim/grad/transformer.blocks.10.ffn.experts.mlp.w2.max":2.9452303351718e-06,"optim/grad/transformer.blocks.1.q_norm.weight.max":2.501502603990957e-05,"optim/grad/transformer.blocks.15.ffn.experts.mlp.w1.norm":0.0024358707014471292,"optim/grad/transformer.blocks.3.k_norm.weight.min":3.6834535421803594e-10,"optim/exp_avg_sq/transformer.blocks.13.attn_out.weight.avg":1.114369881094479e-12,"optim/exp_avg/transformer.blocks.3.ff_norm.weight.max":6.876851330162026e-06,"optim/exp_avg_sq/transformer.blocks.12.ffn.router.layer.weight.max":3.1979743297938512e-09,"optim/param/transformer.blocks.3.att_proj.weight.min":8.715220545241209e-09,"optim/grad/transformer.blocks.13.q_norm.weight.avg":-2.777664143138736e-08,"eval/downstream/arc_challenge_len_norm":0.2441471517086029,"optim/grad/transformer.blocks.8.att_proj.weight.max":0.0005586287006735802,"optim/exp_avg/transformer.blocks.14.q_norm.weight.min":2.0853587953673092e-11,"optim/grad/transformer.blocks.13.ffn.router.layer.weight.avg":-2.9387159372618044e-11,"optim/exp_avg/transformer.blocks.7.att_proj.weight.avg":-3.9449925964829546e-11,"optim/exp_avg/transformer.blocks.7.k_norm.weight.max":1.261068791791331e-05,"optim/exp_avg/transformer.blocks.11.ff_norm.weight.min":4.862531488569743e-10,"optim/exp_avg/transformer.blocks.6.ff_norm.weight.min":1.2959745776530696e-10,"optim/exp_avg/transformer.blocks.1.ff_norm.weight.norm":2.2347161575453356e-05,"optim/grad/transformer.blocks.12.ffn.router.layer.weight.min":3.092281986027956e-11,"optim/param/transformer.blocks.4.ff_norm.weight.min":0.5620664954185486,"optim/exp_avg/transformer.blocks.6.ffn.experts.mlp.w1.norm":0.0006570867262780666,"optim/grad/transformer.blocks.6.k_norm.weight.min":2.5834197003860027e-09,"optim/param/transformer.blocks.2.ffn.router.layer.weight.norm":6.254552364349365,"optim/param/transformer.blocks.2.att_proj.weight.norm":77.55595397949219,"optim/grad/transformer.blocks.10.attn_out.weight.norm":0.003219006583094597,"optim/exp_avg/transformer.blocks.5.ffn.experts.mlp.w2.min":1.1189412171138898e-15,"optim/param/transformer.blocks.14.ff_norm.weight.avg":1.1020724773406982,"optim/grad/transformer.blocks.3.ffn.experts.mlp.w2.min":0,"optim/exp_avg_sq/transformer.blocks.0.k_norm.weight.norm":4.024212907349778e-10,"optim/exp_avg/transformer.blocks.12.attn_out.weight.avg":1.690034223678083e-10,"optim/grad/transformer.blocks.15.attn_out.weight.max":2.81479224213399e-05,"optim/exp_avg/transformer.blocks.3.att_proj.weight.min":1.604418299063458e-14,"eval/dolma_common-crawl-validation/Perplexity":43.30231857299805,"optim/grad/transformer.blocks.0.ffn.router.layer.weight.avg":7.791065570472711e-10,"optim/exp_avg/transformer.blocks.12.attn_norm.weight.max":8.346209142473526e-06,"optim/exp_avg/transformer.blocks.7.attn_out.weight.max":3.857400315609993e-06,"optim/grad/transformer.blocks.14.ffn.router.layer.weight.max":0.00013342685997486115,"optim/grad/transformer.blocks.6.ff_norm.weight.norm":0.0001337029243586585,"optim/param/transformer.blocks.14.q_norm.weight.avg":0.9249089956283569,"optim/grad/transformer.blocks.7.attn_out.weight.max":2.4592314730398357e-05,"optim/grad/transformer.blocks.3.ffn.router.layer.weight.min":1.291482476517558e-10,"optim/param/transformer.blocks.9.ffn.experts.mlp.w1.norm":325.6081237792969,"optim/exp_avg_sq/transformer.blocks.8.ffn.router.layer.weight.max":1.0251451243448173e-08,"optim/exp_avg/transformer.blocks.9.attn_norm.weight.max":1.1705560609698296e-05,"optim/exp_avg_sq/transformer.blocks.14.k_norm.weight.norm":3.459421915597005e-10,"optim/grad/transformer.blocks.2.ffn.experts.mlp.w1.min":0,"optim/exp_avg/transformer.blocks.13.k_norm.weight.min":2.882262831183091e-11,"optim/exp_avg/transformer.blocks.8.attn_norm.weight.norm":5.8135738072451204e-05,"optim/grad/transformer.blocks.7.attn_norm.weight.norm":0.0003126072697341442,"optim/grad/transformer.ln_f.weight.min":1.0622898116707802e-09,"optim/param/transformer.blocks.0.att_proj.weight.avg":-1.0723243804022786e-06,"optim/exp_avg/transformer.blocks.7.ffn.experts.mlp.w2.norm":0.0005713741411454976,"optim/exp_avg_sq/transformer.blocks.12.attn_out.weight.min":1.0853537307179556e-13,"optim/exp_avg_sq/transformer.blocks.10.attn_out.weight.norm":5.97896043785795e-09,"optim/param/transformer.blocks.4.att_proj.weight.max":0.20860055088996887,"optim/exp_avg/transformer.blocks.15.attn_norm.weight.norm":2.6572633942123502e-05,"optim/grad/transformer.ln_f.weight.norm":0.0004997533978894353,"optim/exp_avg_sq/transformer.blocks.10.attn_out.weight.min":1.8742344102488234e-13,"optim/grad/transformer.blocks.3.q_norm.weight.avg":2.595496937374264e-07,"optim/exp_avg/transformer.blocks.5.ff_norm.weight.norm":2.710779881454073e-05,"optim/param/transformer.blocks.9.ff_norm.weight.max":1.1301674842834473,"optim/exp_avg_sq/transformer.blocks.13.att_proj.weight.norm":4.549225351979658e-08,"optim/exp_avg/transformer.blocks.14.ffn.experts.mlp.w1.min":1.5334390662538275e-16,"optim/exp_avg_sq/transformer.blocks.0.ffn.router.layer.weight.max":4.227847938409468e-08,"eval/downstream/mmlu_social_sciences_mc_5shot_test_len_norm":0.23204420506954193,"optim/param/transformer.blocks.13.q_norm.weight.max":1.374994158744812,"optim/exp_avg/transformer.blocks.12.ff_norm.weight.max":3.941013346775435e-06,"optim/param/transformer.blocks.4.attn_out.weight.avg":-4.230358172208071e-06,"optim/grad/transformer.blocks.4.ffn.experts.mlp.w2.min":0,"optim/param/transformer.blocks.12.att_proj.weight.avg":-1.0624772585288156e-05,"optim/grad/transformer.blocks.7.attn_out.weight.avg":-8.353346903078318e-10,"optim/exp_avg_sq/transformer.blocks.5.attn_norm.weight.min":1.1767625068825893e-11,"optim/exp_avg/transformer.blocks.8.ffn.experts.mlp.w1.norm":0.0006644146633334458,"optim/exp_avg_sq/transformer.blocks.0.ffn.router.layer.weight.norm":1.148878894241534e-07,"optim/grad/transformer.blocks.5.att_proj.weight.min":0,"optim/exp_avg/transformer.blocks.11.attn_norm.weight.avg":6.544713926359691e-08,"optim/exp_avg_sq/transformer.blocks.1.attn_norm.weight.max":8.190543354658075e-08,"optim/exp_avg/transformer.blocks.9.att_proj.weight.avg":4.117940710646195e-11,"optim/param/transformer.blocks.3.ffn.experts.mlp.w2.max":0.36061418056488037,"optim/exp_avg_sq/transformer.blocks.2.att_proj.weight.avg":2.6189611243565558e-11,"optim/exp_avg/transformer.blocks.6.attn_out.weight.avg":-1.3786483066269284e-10,"optim/grad/transformer.blocks.3.att_proj.weight.min":0,"optim/grad/transformer.blocks.10.ffn.router.layer.weight.min":1.0550138540565968e-10,"optim/exp_avg/transformer.blocks.0.ffn.experts.mlp.w1.norm":0.00024135679996106774,"optim/param/transformer.blocks.8.q_norm.weight.norm":44.04863739013672,"throughput/total_training_log_Gflops":26.952063662887607,"optim/exp_avg/transformer.ln_f.weight.max":2.6112338673556224e-05,"optim/exp_avg_sq/transformer.blocks.3.attn_norm.weight.avg":1.1253928344778785e-10,"optim/exp_avg_sq/transformer.blocks.14.ffn.router.layer.weight.min":6.6003595818042715e-12,"optim/exp_avg/transformer.blocks.8.ffn.experts.mlp.w1.avg":5.149742611509911e-12,"optim/exp_avg/transformer.blocks.7.q_norm.weight.norm":3.732026380021125e-05,"optim/grad/transformer.blocks.13.ffn.router.layer.weight.norm":0.0025913314893841743,"optim/exp_avg/transformer.blocks.14.k_norm.weight.norm":1.4384226233232766e-05,"optim/exp_avg/transformer.blocks.9.attn_out.weight.max":3.435575536059332e-06,"optim/grad/transformer.blocks.6.ffn.experts.mlp.w2.max":4.461588559934171e-06,"optim/grad/transformer.blocks.4.att_proj.weight.norm":0.017500419169664383,"optim/grad/transformer.blocks.9.ffn.experts.mlp.w1.min":0,"optim/exp_avg/transformer.blocks.7.ff_norm.weight.avg":-1.6577112660343118e-07,"optim/param/transformer.blocks.9.q_norm.weight.avg":0.9460270404815674,"throughput/device/tokens_per_second":28196.55516066193,"optim/exp_avg/transformer.blocks.12.q_norm.weight.norm":2.147642953786999e-05,"optim/exp_avg_sq/transformer.blocks.14.q_norm.weight.min":1.1700116922400486e-15,"optim/grad/transformer.blocks.4.att_proj.weight.avg":-2.8387894257519974e-09,"optim/grad/transformer.blocks.7.q_norm.weight.max":3.781800114666112e-05,"optim/exp_avg_sq/transformer.blocks.11.q_norm.weight.min":8.16980133582423e-15,"optim/exp_avg/transformer.blocks.10.q_norm.weight.avg":-1.7683419173408765e-07,"optim/grad/transformer.blocks.6.q_norm.weight.max":5.826168126077391e-05,"optim/grad/transformer.blocks.1.ffn.router.layer.weight.max":0.000293865567073226,"optim/exp_avg/transformer.blocks.10.attn_norm.weight.norm":4.329844523454085e-05,"optim/exp_avg_sq/transformer.blocks.6.attn_norm.weight.norm":3.766007239391911e-08,"optim/exp_avg_sq/transformer.blocks.11.attn_out.weight.min":1.4091639910073644e-13,"optim/grad/transformer.blocks.13.attn_norm.weight.max":3.984104114351794e-05,"optim/exp_avg_sq/transformer.blocks.6.q_norm.weight.min":4.9276798157053044e-15,"optim/grad/transformer.blocks.14.ffn.experts.mlp.w1.min":0,"optim/param/transformer.blocks.10.attn_out.weight.min":5.066322472657703e-09,"optim/exp_avg/transformer.blocks.0.k_norm.weight.min":7.365731288788702e-12,"optim/param/transformer.blocks.0.ffn.experts.mlp.w1.norm":289.4080810546875,"optim/exp_avg_sq/transformer.blocks.5.k_norm.weight.norm":3.3108165098383324e-08,"optim/exp_avg_sq/transformer.blocks.4.k_norm.weight.max":6.126660068161982e-09,"optim/grad/transformer.blocks.6.ffn.experts.mlp.w2.norm":0.002440261421725154,"optim/exp_avg_sq/transformer.blocks.15.ffn.router.layer.weight.min":3.756424858669671e-12,"optim/grad/transformer.blocks.10.attn_norm.weight.min":1.2732925824820995e-09,"optim/exp_avg_sq/transformer.blocks.9.ffn.router.layer.weight.min":4.943982723215612e-12,"optim/grad/transformer.blocks.15.q_norm.weight.max":1.7687911167740822e-05,"optim/exp_avg/transformer.blocks.1.k_norm.weight.norm":2.1490277504199184e-05,"optim/param/transformer.blocks.13.att_proj.weight.min":9.646807797025758e-10,"optim/grad/transformer.blocks.15.att_proj.weight.avg":2.9529888950774463e-11,"optim/exp_avg/transformer.blocks.1.ffn.experts.mlp.w1.norm":0.0005291158449836075,"optim/param/transformer.blocks.14.ff_norm.weight.max":1.219128131866455,"optim/grad/transformer.blocks.15.attn_norm.weight.norm":0.000162024240125902,"optim/exp_avg_sq/transformer.blocks.2.ff_norm.weight.min":2.0494749560645564e-12,"optim/param/transformer.blocks.3.ffn.experts.mlp.w2.norm":307.6181335449219,"optim/exp_avg_sq/transformer.blocks.13.ff_norm.weight.norm":3.6006550518941083e-10,"optim/grad/transformer.blocks.10.ffn.experts.mlp.w2.min":0,"optim/exp_avg/transformer.blocks.4.attn_out.weight.min":1.5195768363236717e-13,"optim/grad/transformer.blocks.4.ffn.experts.mlp.w1.norm":0.0035938445944339037,"optim/exp_avg_sq/transformer.blocks.11.att_proj.weight.min":1.992180834360646e-15,"optim/exp_avg_sq/transformer.blocks.3.ffn.experts.mlp.w2.norm":4.096487149496397e-09,"optim/exp_avg/transformer.blocks.2.attn_out.weight.max":1.2940486158186104e-05,"optim/exp_avg/transformer.blocks.10.ff_norm.weight.norm":1.908547528728377e-05,"optim/grad/transformer.blocks.2.ffn.router.layer.weight.norm":0.002826735842972994,"optim/exp_avg/transformer.blocks.2.ffn.experts.mlp.w2.min":1.842103960282603e-16,"optim/grad/transformer.blocks.3.ffn.experts.mlp.w1.min":0,"optim/grad/transformer.blocks.9.attn_out.weight.avg":-1.065379562348312e-09,"optim/exp_avg/transformer.blocks.15.ff_norm.weight.max":1.6972770026768558e-05,"optim/param/transformer.blocks.3.ffn.experts.mlp.w1.avg":2.6601213903632015e-05,"optim/exp_avg_sq/transformer.blocks.6.attn_norm.weight.min":9.844746545750738e-12,"optim/exp_avg/transformer.blocks.15.ffn.experts.mlp.w2.min":1.7581418193860204e-16,"optim/exp_avg_sq/transformer.blocks.15.q_norm.weight.max":1.920833769819552e-10,"optim/exp_avg_sq/transformer.blocks.7.attn_norm.weight.min":7.55972454774545e-12,"optim/param/transformer.blocks.7.ff_norm.weight.avg":0.9862669706344604,"optim/exp_avg_sq/transformer.blocks.4.attn_out.weight.max":1.1199902116842964e-09,"optim/exp_avg/transformer.blocks.11.ffn.experts.mlp.w2.avg":4.8650819484130636e-11,"optim/grad/transformer.blocks.10.att_proj.weight.min":0,"optim/param/transformer.blocks.0.attn_out.weight.max":0.2669259309768677,"optim/grad/transformer.blocks.0.ffn.experts.mlp.w1.norm":0.0011105538578704,"optim/exp_avg/transformer.blocks.12.ffn.router.layer.weight.avg":-1.0979384068576792e-10,"optim/exp_avg_sq/transformer.blocks.15.attn_norm.weight.norm":5.842113903753443e-09,"optim/exp_avg/transformer.blocks.15.k_norm.weight.min":6.512192174401576e-11,"optim/grad/transformer.blocks.4.ff_norm.weight.min":9.360974217997864e-10,"optim/exp_avg/transformer.blocks.9.ffn.experts.mlp.w1.min":8.336738612475607e-16,"optim/exp_avg_sq/transformer.blocks.8.q_norm.weight.norm":1.2207767241534384e-08,"optim/param/transformer.blocks.7.att_proj.weight.min":1.0939219530214928e-09,"optim/grad/transformer.blocks.2.ffn.experts.mlp.w2.min":0,"optim/exp_avg_sq/transformer.blocks.13.q_norm.weight.min":2.274481669087454e-15,"optim/exp_avg/transformer.blocks.13.ff_norm.weight.norm":1.8745247871265747e-05,"optim/param/transformer.blocks.13.attn_out.weight.avg":-2.355781543883495e-05,"optim/grad/transformer.blocks.3.k_norm.weight.norm":0.00014095463848207146,"optim/exp_avg_sq/transformer.blocks.0.k_norm.weight.min":1.6594620575144207e-14,"optim/exp_avg/transformer.blocks.6.ffn.experts.mlp.w2.min":1.7489331948208268e-15,"optim/grad/transformer.blocks.3.ffn.router.layer.weight.max":0.0003930210368707776,"optim/param/transformer.blocks.14.ffn.experts.mlp.w2.avg":-1.3579583537648432e-06,"optim/grad/transformer.blocks.8.ffn.experts.mlp.w1.max":7.485358946723863e-05,"optim/grad/transformer.blocks.0.att_proj.weight.norm":0.008246228098869324,"optim/grad/transformer.blocks.9.ffn.router.layer.weight.norm":0.0026186152826994658,"optim/exp_avg/transformer.blocks.1.att_proj.weight.norm":0.003384284907951951,"optim/exp_avg/transformer.blocks.11.ff_norm.weight.norm":1.9298133338452317e-05,"optim/exp_avg_sq/transformer.blocks.0.att_proj.weight.max":3.394841385784275e-08,"optim/grad/transformer.blocks.14.att_proj.weight.max":9.364965808345005e-05,"optim/param/transformer.blocks.15.attn_out.weight.max":0.322346568107605,"optim/exp_avg/transformer.blocks.3.ffn.experts.mlp.w1.min":1.6556118692526763e-15,"optim/exp_avg_sq/transformer.wte.weight.min":0,"optim/param/transformer.blocks.14.ffn.router.layer.weight.min":2.0382421439535392e-07,"optim/exp_avg/transformer.blocks.0.ffn.experts.mlp.w2.avg":-5.161148244892111e-11,"optim/exp_avg_sq/transformer.blocks.15.ffn.experts.mlp.w1.max":1.0589660259796574e-09,"optim/grad/transformer.blocks.10.ffn.experts.mlp.w2.avg":9.350915874950516e-11,"optim/grad/transformer.blocks.8.ff_norm.weight.avg":2.0030437042350968e-07,"optim/exp_avg/transformer.blocks.7.k_norm.weight.avg":-1.7514017258690728e-07,"optim/exp_avg/transformer.blocks.2.att_proj.weight.avg":2.1418490592228778e-10,"optim/exp_avg_sq/transformer.blocks.11.ffn.router.layer.weight.norm":2.5937630709904624e-08,"optim/grad/transformer.blocks.5.ff_norm.weight.max":4.169442399870604e-05,"optim/exp_avg/transformer.blocks.10.attn_norm.weight.avg":6.79727776287109e-08,"optim/exp_avg_sq/transformer.blocks.9.attn_out.weight.min":2.5032810568349773e-13,"optim/exp_avg/transformer.blocks.13.attn_norm.weight.min":3.23108734212596e-10,"optim/param/transformer.blocks.0.ffn.router.layer.weight.max":0.05853075161576271,"optim/exp_avg/transformer.blocks.2.ffn.router.layer.weight.avg":5.289761961790873e-10,"optim/exp_avg/transformer.blocks.14.ffn.experts.mlp.w2.norm":0.00039293812005780637,"optim/param/transformer.blocks.14.att_proj.weight.min":3.4801789228211533e-10,"optim/param/transformer.blocks.13.ffn.experts.mlp.w2.avg":-2.3181976303021656e-06,"optim/exp_avg/transformer.blocks.11.k_norm.weight.avg":-1.1954197987051884e-07,"optim/exp_avg/transformer.blocks.2.ff_norm.weight.norm":2.8402946554706432e-05,"optim/exp_avg/transformer.blocks.0.ff_norm.weight.min":1.0981226344908279e-10,"optim/param/transformer.blocks.11.attn_norm.weight.max":0.6829401850700378,"optim/exp_avg/transformer.blocks.2.attn_out.weight.min":6.416219687716343e-14,"optim/grad/transformer.blocks.4.ff_norm.weight.norm":0.000127217557746917,"optim/param/transformer.blocks.0.ff_norm.weight.min":0.3865184783935547,"optim/param/transformer.blocks.6.ff_norm.weight.min":0.5721771717071533,"optim/grad/transformer.blocks.8.ffn.router.layer.weight.max":0.00023504052660427988,"optim/exp_avg/transformer.blocks.4.attn_out.weight.max":1.0288906196365133e-05,"optim/grad/transformer.blocks.7.att_proj.weight.min":2.842170943040401e-14,"optim/param/transformer.blocks.1.att_proj.weight.max":0.20338261127471924,"optim/exp_avg_sq/transformer.blocks.10.k_norm.weight.avg":8.158222361553946e-12,"optim/grad/transformer.blocks.7.ffn.router.layer.weight.max":0.00011525499576237053,"optim/grad/transformer.blocks.1.k_norm.weight.avg":1.592379987869208e-07,"optim/exp_avg_sq/transformer.blocks.8.attn_norm.weight.max":1.1580207903705286e-08,"optim/exp_avg_sq/transformer.blocks.1.attn_norm.weight.norm":1.2197294552152016e-07,"optim/grad/transformer.blocks.5.attn_out.weight.avg":-8.541398699435376e-10,"optim/exp_avg/transformer.blocks.2.ff_norm.weight.max":1.046349098032806e-05,"optim/grad/transformer.blocks.12.ff_norm.weight.max":1.4682456821901724e-05,"optim/exp_avg/transformer.blocks.12.attn_out.weight.norm":0.0005071396008133888,"optim/param/transformer.blocks.14.k_norm.weight.max":1.4039264917373657,"optim/exp_avg_sq/transformer.blocks.7.q_norm.weight.min":2.8146653374354184e-17,"optim/exp_avg_sq/transformer.blocks.6.attn_out.weight.min":5.249980138122279e-13,"optim/exp_avg/transformer.blocks.8.q_norm.weight.max":6.658681741100736e-06,"optim/exp_avg/transformer.blocks.4.attn_norm.weight.norm":6.40399448457174e-05,"optim/exp_avg_sq/transformer.blocks.8.k_norm.weight.norm":9.000111766965802e-09,"optim/grad/transformer.blocks.5.ff_norm.weight.min":2.964952727779746e-10,"optim/grad/transformer.blocks.3.attn_out.weight.avg":-6.86983248066042e-10,"optim/exp_avg/transformer.blocks.0.k_norm.weight.avg":1.9570499887322512e-08,"optim/exp_avg_sq/transformer.blocks.8.q_norm.weight.max":1.1369067465238913e-08,"optim/param/transformer.blocks.12.attn_out.weight.max":0.17535698413848877,"optim/grad/transformer.blocks.15.q_norm.weight.min":2.5778490453376435e-11,"optim/param/transformer.blocks.14.attn_out.weight.avg":-1.2281729141250253e-05,"optim/exp_avg/transformer.blocks.6.att_proj.weight.min":6.8293030871255725e-15,"optim/param/transformer.blocks.15.attn_norm.weight.avg":0.6402726173400879,"optim/param/transformer.blocks.5.ffn.experts.mlp.w1.max":0.17054636776447296,"optim/exp_avg/transformer.blocks.15.attn_norm.weight.avg":5.437059513724307e-08,"optim/exp_avg_sq/transformer.blocks.6.ff_norm.weight.avg":9.516492628647288e-12,"optim/exp_avg/transformer.blocks.7.ff_norm.weight.norm":3.125709554296918e-05,"eval/downstream/commonsense_qa_len_norm":0.2694512605667114,"optim/exp_avg_sq/transformer.blocks.6.att_proj.weight.min":1.2935556827618246e-15,"optim/exp_avg_sq/transformer.blocks.3.k_norm.weight.min":1.159511956529179e-14,"optim/exp_avg/transformer.blocks.3.attn_norm.weight.avg":2.996549710587715e-07,"optim/param/transformer.blocks.7.q_norm.weight.norm":40.89651107788086,"optim/param/transformer.blocks.7.ffn.experts.mlp.w2.norm":320.7547607421875,"optim/exp_avg_sq/transformer.blocks.14.q_norm.weight.norm":3.474839860295731e-10,"optim/exp_avg_sq/transformer.blocks.1.ff_norm.weight.min":1.1787349525269053e-12,"optim/exp_avg_sq/transformer.blocks.9.att_proj.weight.min":1.4249789855171969e-15,"optim/exp_avg_sq/transformer.blocks.11.ffn.experts.mlp.w1.max":9.27200191624955e-11,"optim/grad/transformer.blocks.2.ff_norm.weight.avg":-3.1115928322833497e-07,"optim/exp_avg/transformer.blocks.4.att_proj.weight.avg":2.9716132332602285e-10,"optim/param/transformer.blocks.4.ffn.router.layer.weight.norm":6.5981669425964355,"optim/exp_avg_sq/transformer.blocks.8.ffn.experts.mlp.w1.min":2.141207387584094e-15,"optim/grad/transformer.blocks.4.ff_norm.weight.max":3.872660454362631e-05,"optim/exp_avg_sq/transformer.blocks.1.ffn.router.layer.weight.avg":5.175677247892807e-11,"optim/exp_avg_sq/transformer.blocks.7.ff_norm.weight.norm":1.4004932769040579e-09,"train/masked_instances_local_rank":0,"optim/param/transformer.blocks.0.attn_norm.weight.norm":14.981907844543457,"optim/param/transformer.blocks.4.q_norm.weight.avg":1.0030491352081299,"optim/param/transformer.blocks.11.q_norm.weight.max":1.4174020290374756,"optim/exp_avg_sq/transformer.blocks.4.ffn.experts.mlp.w2.norm":8.817421237594658e-10,"optim/grad/transformer.blocks.8.ffn.router.layer.weight.norm":0.0027070159558206797,"optim/exp_avg/transformer.blocks.1.ffn.experts.mlp.w2.norm":0.0006173649453558028,"optim/param/transformer.blocks.3.att_proj.weight.norm":77.86202239990234,"optim/param/transformer.blocks.2.ff_norm.weight.norm":39.63054275512695,"optim/grad/transformer.blocks.13.attn_out.weight.norm":0.0021511996164917946,"optim/exp_avg/transformer.blocks.15.attn_out.weight.norm":0.0005308098043315113,"optim/param/transformer.blocks.14.attn_norm.weight.norm":29.579891204833984,"optim/param/transformer.blocks.3.att_proj.weight.avg":-8.380587246392679e-07,"optim/param/transformer.blocks.11.ffn.router.layer.weight.avg":0.0002424318081466481,"optim/exp_avg_sq/transformer.blocks.7.att_proj.weight.norm":8.91820889137307e-07,"optim/grad/transformer.blocks.4.attn_norm.weight.min":3.6980054574087262e-09,"optim/param/transformer.blocks.11.ffn.experts.mlp.w1.norm":330.4459228515625,"optim/grad/transformer.blocks.12.ffn.experts.mlp.w1.min":0,"optim/exp_avg_sq/transformer.blocks.2.ffn.experts.mlp.w2.norm":1.5544336928741131e-09,"optim/exp_avg/transformer.blocks.11.q_norm.weight.max":4.185018042335287e-06,"optim/grad/transformer.blocks.2.ffn.experts.mlp.w1.avg":-3.1478441381294076e-11,"optim/exp_avg_sq/transformer.blocks.4.k_norm.weight.avg":5.5580942626143326e-11,"optim/param/transformer.blocks.6.attn_out.weight.min":5.966827032466426e-09,"optim/exp_avg_sq/transformer.blocks.10.attn_norm.weight.max":2.1440351716250916e-09,"optim/exp_avg_sq/transformer.blocks.1.attn_out.weight.avg":5.462509351100708e-12,"optim/grad/transformer.blocks.10.q_norm.weight.min":1.0186340659856796e-10,"optim/exp_avg/transformer.blocks.0.attn_out.weight.min":3.236794784023528e-13,"optim/exp_avg/transformer.blocks.3.k_norm.weight.max":3.65976143257285e-06,"optim/param/transformer.blocks.9.ffn.router.layer.weight.min":4.3284637740725884e-07,"optim/exp_avg/transformer.blocks.9.att_proj.weight.norm":0.0014304996002465487,"eval/downstream/mmlu_other_mc_5shot_test_len_norm":0.23781616985797882,"optim/exp_avg/transformer.blocks.4.ffn.experts.mlp.w1.min":5.48792328888706e-16,"optim/param/transformer.blocks.2.attn_out.weight.min":3.3900504625705707e-09,"optim/exp_avg_sq/transformer.blocks.10.attn_norm.weight.min":5.7525512131562095e-12,"optim/exp_avg_sq/transformer.blocks.8.ff_norm.weight.avg":6.347449042071185e-12,"optim/param/transformer.blocks.8.ffn.experts.mlp.w1.max":0.1777818650007248,"optim/exp_avg/transformer.blocks.8.attn_norm.weight.min":2.447409497285946e-10,"optim/exp_avg/transformer.blocks.1.attn_out.weight.norm":0.0010215614456683397,"optim/exp_avg_sq/transformer.blocks.12.ff_norm.weight.norm":3.3126490439627787e-10,"optim/param/transformer.blocks.0.q_norm.weight.max":1.2084630727767944,"optim/param/transformer.blocks.15.attn_norm.weight.min":0.577773928642273,"eval/dolma_books-validation/Perplexity":40.4458122253418,"optim/exp_avg/transformer.blocks.5.ffn.experts.mlp.w2.avg":-4.5879099130896606e-11,"optim/exp_avg/transformer.blocks.7.attn_norm.weight.norm":6.394255615305156e-05,"optim/grad/transformer.blocks.0.attn_out.weight.avg":1.179154274222327e-10,"optim/param/transformer.blocks.0.ff_norm.weight.avg":0.6857038140296936,"optim/grad/transformer.blocks.15.q_norm.weight.norm":7.920833013486117e-05,"optim/exp_avg/transformer.blocks.15.ffn.router.layer.weight.norm":0.000493282510433346,"optim/exp_avg/transformer.blocks.1.attn_out.weight.avg":3.144640797758669e-11,"optim/param/transformer.blocks.0.att_proj.weight.min":1.2718168740377678e-09,"optim/grad/transformer.blocks.10.attn_out.weight.max":1.3347133062779903e-05,"optim/grad/transformer.blocks.2.ffn.experts.mlp.w2.avg":1.2494220935632683e-10,"optim/param/transformer.blocks.1.ffn.experts.mlp.w1.min":3.015369204328877e-11,"optim/exp_avg/transformer.blocks.10.attn_out.weight.norm":0.0007237464305944741,"optim/param/transformer.blocks.7.attn_out.weight.max":0.12075550109148026,"optim/exp_avg/transformer.blocks.14.k_norm.weight.max":2.5376234589202795e-06,"optim/param/transformer.blocks.1.q_norm.weight.min":0.5439584851264954,"optim/param/transformer.blocks.10.attn_out.weight.max":0.16531415283679962,"optim/param/transformer.blocks.6.ff_norm.weight.avg":0.9509967565536499,"optim/exp_avg_sq/transformer.blocks.14.ffn.experts.mlp.w1.min":7.521938445237885e-17,"optim/exp_avg/transformer.blocks.12.ffn.experts.mlp.w2.avg":5.1241573484350766e-11,"optim/exp_avg/transformer.blocks.15.q_norm.weight.avg":-4.6361414263174083e-08,"optim/param/transformer.blocks.1.ffn.router.layer.weight.max":0.08796732872724533,"optim/exp_avg/transformer.blocks.11.attn_norm.weight.norm":3.779435792239383e-05,"optim/grad/transformer.blocks.14.ff_norm.weight.max":1.035858440445736e-05,"optim/param/transformer.blocks.4.k_norm.weight.min":0.3460426926612854,"optim/param/transformer.blocks.9.ff_norm.weight.norm":47.51114273071289,"optim/param/transformer.blocks.2.ff_norm.weight.max":1.0605367422103882,"optim/grad/transformer.blocks.1.att_proj.weight.norm":0.021344847977161407,"eval/downstream/copa_acc":0.5299999713897705,"optim/exp_avg_sq/transformer.blocks.0.att_proj.weight.avg":5.015805912972349e-12,"optim/exp_avg_sq/transformer.blocks.5.ff_norm.weight.min":2.681230020992742e-12,"optim/exp_avg/transformer.blocks.6.attn_norm.weight.min":1.766215507181812e-10,"optim/param/transformer.blocks.13.ff_norm.weight.avg":1.1385713815689087,"optim/param/transformer.blocks.13.ffn.experts.mlp.w2.min":1.694079945768756e-11,"optim/exp_avg_sq/transformer.blocks.12.ff_norm.weight.avg":2.9911724139242146e-12,"optim/exp_avg/transformer.blocks.12.attn_norm.weight.avg":2.32378187803306e-08,"optim/exp_avg/transformer.blocks.11.ffn.router.layer.weight.avg":-1.1948431133390613e-10,"optim/exp_avg_sq/transformer.blocks.7.att_proj.weight.max":8.835198883616613e-08,"optim/param/transformer.blocks.13.attn_norm.weight.max":0.7452594637870789,"optim/param/transformer.blocks.8.k_norm.weight.avg":0.9583319425582886,"optim/param/transformer.blocks.9.ffn.experts.mlp.w2.max":0.27859559655189514,"optim/exp_avg/transformer.blocks.12.k_norm.weight.norm":2.1529389414354227e-05,"optim/exp_avg/transformer.blocks.11.attn_norm.weight.max":1.1062017620133702e-05,"optim/param/transformer.blocks.6.q_norm.weight.max":1.3204141855239868,"optim/grad/transformer.blocks.0.attn_out.weight.max":9.318863158114254e-05,"optim/exp_avg_sq/transformer.blocks.12.ffn.experts.mlp.w1.max":3.993462852069918e-11,"optim/grad/transformer.blocks.9.ffn.experts.mlp.w1.avg":3.0285088672976634e-12,"optim/param/transformer.blocks.10.q_norm.weight.norm":44.206722259521484,"optim/exp_avg/transformer.blocks.6.ffn.router.layer.weight.avg":1.4705703321737928e-10,"optim/exp_avg_sq/transformer.blocks.8.attn_norm.weight.min":8.217347809147402e-12,"optim/grad/transformer.blocks.6.ff_norm.weight.avg":-1.3310761914908653e-07,"optim/grad/transformer.blocks.3.attn_norm.weight.min":7.057678885757923e-10,"optim/exp_avg_sq/transformer.ff_out.weight.max":1.2997739986531087e-06,"optim/param/transformer.blocks.1.attn_norm.weight.avg":0.3879379630088806,"optim/exp_avg_sq/transformer.blocks.11.ff_norm.weight.max":2.788283215426901e-10,"optim/grad/transformer.blocks.3.ff_norm.weight.min":6.912159733474255e-10,"optim/exp_avg/transformer.blocks.15.k_norm.weight.max":1.4949782780604437e-06,"optim/exp_avg/transformer.blocks.3.ffn.router.layer.weight.norm":0.0006020973087288439,"optim/exp_avg/transformer.blocks.14.attn_norm.weight.norm":2.436274371575564e-05,"optim/grad/transformer.blocks.9.att_proj.weight.avg":-1.2624512546466349e-11,"optim/exp_avg/transformer.blocks.0.att_proj.weight.max":6.612732977373526e-05,"optim/param/transformer.blocks.5.attn_out.weight.max":0.12472578138113022,"optim/exp_avg/transformer.blocks.10.ffn.experts.mlp.w1.min":5.587850942048202e-16,"optim/grad/transformer.blocks.10.attn_norm.weight.max":6.712557660648599e-05,"optim/param/transformer.blocks.2.q_norm.weight.max":1.2583458423614502,"eval/dolma_stack-validation/Perplexity":3.845301628112793,"optim/param/transformer.blocks.8.ff_norm.weight.avg":1.0054819583892822,"optim/exp_avg/transformer.blocks.15.ff_norm.weight.norm":3.608913175412454e-05,"optim/grad/transformer.blocks.0.ffn.router.layer.weight.min":3.637978807091713e-12,"optim/param/transformer.blocks.5.ffn.router.layer.weight.avg":0.0004157812218181789,"optim/param/transformer.blocks.5.ffn.router.layer.weight.norm":6.622762680053711,"optim/param/transformer.blocks.7.k_norm.weight.min":0.31262120604515076,"optim/param/transformer.blocks.10.ff_norm.weight.min":0.5913981199264526,"optim/grad/transformer.blocks.12.attn_norm.weight.norm":0.00014853230095468462,"optim/grad/transformer.blocks.3.ffn.router.layer.weight.avg":6.643343652967815e-10,"optim/param/transformer.wte.weight.min":8.388665317937694e-10,"optim/exp_avg/transformer.blocks.13.ff_norm.weight.min":1.2970911844600863e-10,"optim/exp_avg/transformer.blocks.9.attn_norm.weight.avg":1.4591218189252686e-07,"optim/exp_avg_sq/transformer.blocks.14.att_proj.weight.min":4.47966185528764e-16,"optim/param/transformer.blocks.6.ffn.router.layer.weight.avg":0.00035383296199142933,"optim/grad/transformer.blocks.12.ffn.experts.mlp.w2.norm":0.001881940639577806,"optim/param/transformer.blocks.6.k_norm.weight.avg":0.959283709526062,"optim/exp_avg/transformer.blocks.5.k_norm.weight.min":2.7737024482776818e-11,"optim/grad/transformer.blocks.11.k_norm.weight.min":1.554099071654491e-10,"optim/param/transformer.blocks.12.ffn.experts.mlp.w1.min":3.5876931980816096e-10,"optim/param/transformer.blocks.9.k_norm.weight.min":0.22603438794612885,"optim/param/transformer.blocks.6.ffn.router.layer.weight.norm":6.8482255935668945,"optim/exp_avg/transformer.blocks.8.att_proj.weight.min":1.7847898994231143e-14,"optim/grad/transformer.blocks.13.ff_norm.weight.avg":-2.6351273163527367e-08,"optim/exp_avg/transformer.blocks.7.ffn.experts.mlp.w1.avg":8.227734465959813e-12,"optim/param/transformer.blocks.13.ffn.experts.mlp.w1.max":0.39510348439216614,"optim/exp_avg/transformer.blocks.13.ffn.router.layer.weight.avg":-6.57538468118446e-11,"optim/exp_avg_sq/transformer.blocks.12.att_proj.weight.norm":5.362043609125067e-08,"optim/exp_avg_sq/transformer.blocks.3.ff_norm.weight.norm":3.848827923036424e-09,"optim/exp_avg/transformer.blocks.6.q_norm.weight.min":6.404642788337966e-11,"optim/exp_avg/transformer.blocks.12.ffn.experts.mlp.w2.min":1.897168513392421e-17,"optim/exp_avg_sq/transformer.blocks.8.attn_out.weight.avg":4.959281249550251e-12,"optim/exp_avg_sq/transformer.blocks.12.k_norm.weight.min":6.165524447460319e-15,"optim/exp_avg_sq/transformer.blocks.1.ffn.router.layer.weight.min":2.3026025530725747e-12,"optim/exp_avg/transformer.blocks.8.ffn.router.layer.weight.norm":0.0006088727386668324,"optim/grad/transformer.blocks.9.ffn.experts.mlp.w2.avg":1.0331123456719382e-10,"optim/param/transformer.blocks.5.attn_norm.weight.avg":0.5919744372367859,"optim/exp_avg_sq/transformer.blocks.0.attn_out.weight.norm":6.962699217183399e-08,"optim/param/transformer.blocks.0.k_norm.weight.norm":36.77031707763672,"optim/exp_avg/transformer.blocks.4.ffn.experts.mlp.w1.norm":0.0007432793499901891,"optim/exp_avg_sq/transformer.blocks.3.att_proj.weight.norm":3.191818905179389e-06,"optim/exp_avg/transformer.blocks.9.q_norm.weight.min":7.1166744372574975e-12,"optim/exp_avg/transformer.blocks.6.attn_norm.weight.norm":5.963683724985458e-05,"optim/exp_avg_sq/transformer.blocks.6.ffn.experts.mlp.w1.min":5.329383073358288e-16,"optim/param/transformer.blocks.7.att_proj.weight.avg":-4.5508895709645e-06,"optim/exp_avg_sq/transformer.blocks.9.ffn.experts.mlp.w1.min":3.9381240459548114e-16,"optim/exp_avg/transformer.blocks.2.attn_norm.weight.norm":0.0001223069557454437,"optim/exp_avg/transformer.blocks.5.att_proj.weight.max":0.00010013666906161234,"optim/grad/transformer.blocks.10.att_proj.weight.norm":0.006255653221160173,"optim/exp_avg_sq/transformer.blocks.15.q_norm.weight.avg":3.2499805797947534e-12,"optim/grad/transformer.blocks.10.att_proj.weight.avg":-2.4467722292698113e-10,"optim/exp_avg/transformer.blocks.9.ffn.experts.mlp.w2.max":1.060743329617253e-06,"optim/exp_avg_sq/transformer.blocks.15.ff_norm.weight.min":2.5449503615604385e-12,"optim/exp_avg_sq/transformer.blocks.4.attn_norm.weight.max":7.745159535943458e-08,"optim/param/transformer.blocks.9.ffn.experts.mlp.w1.avg":2.7682206564350054e-05,"optim/exp_avg/transformer.blocks.9.ffn.experts.mlp.w1.max":2.072579718515044e-06,"optim/exp_avg_sq/transformer.blocks.12.ffn.router.layer.weight.norm":2.3666830273327832e-08,"optim/grad/transformer.blocks.1.ff_norm.weight.max":4.0047510992735624e-05,"optim/grad/transformer.blocks.1.ffn.router.layer.weight.avg":8.681606544769238e-10,"optim/exp_avg/transformer.blocks.2.ffn.experts.mlp.w2.avg":-1.6719877218851487e-11,"optim/grad/transformer.blocks.7.att_proj.weight.avg":-5.695071081390779e-10,"optim/param/transformer.blocks.8.att_proj.weight.max":0.18117406964302063,"optim/exp_avg_sq/transformer.blocks.7.ffn.router.layer.weight.avg":5.970252070497395e-11,"optim/exp_avg_sq/transformer.blocks.1.attn_out.weight.min":2.09868919660329e-13,"optim/grad/transformer.blocks.3.ffn.experts.mlp.w2.avg":-5.895599564098575e-10,"optim/param/transformer.blocks.12.ffn.experts.mlp.w1.avg":9.539430720906239e-06,"optim/exp_avg_sq/transformer.blocks.1.k_norm.weight.norm":7.745190599983687e-10,"optim/param/transformer.blocks.1.ffn.experts.mlp.w1.max":0.18183526396751404,"optim/grad/transformer.blocks.15.att_proj.weight.min":0,"optim/exp_avg/transformer.blocks.0.q_norm.weight.norm":1.610177605471108e-05,"optim/exp_avg/transformer.blocks.8.ff_norm.weight.avg":-1.3042000546192867e-07,"optim/param/transformer.blocks.13.ff_norm.weight.max":1.225967526435852,"optim/exp_avg_sq/transformer.blocks.5.ffn.router.layer.weight.max":2.584798863836113e-08,"optim/grad/transformer.blocks.12.attn_out.weight.avg":-1.262932813883566e-11,"optim/param/transformer.blocks.5.k_norm.weight.min":0.46852055191993713,"optim/grad/transformer.blocks.11.ff_norm.weight.avg":2.070495384032256e-07,"optim/exp_avg_sq/transformer.blocks.7.q_norm.weight.avg":1.504887707504121e-11,"optim/exp_avg_sq/transformer.blocks.14.attn_out.weight.min":5.003347725279078e-14,"optim/exp_avg/transformer.blocks.6.ffn.experts.mlp.w2.norm":0.0005525619490072131,"optim/exp_avg_sq/transformer.blocks.13.att_proj.weight.avg":1.2278900769421841e-12,"optim/exp_avg_sq/transformer.blocks.10.ffn.router.layer.weight.max":3.944877757788845e-09,"optim/param/transformer.blocks.8.ffn.experts.mlp.w1.min":6.950934619553983e-12,"optim/exp_avg/transformer.blocks.3.ffn.experts.mlp.w2.max":2.990973598571145e-06,"optim/exp_avg/transformer.blocks.6.ffn.experts.mlp.w2.avg":3.03361294418103e-11,"optim/exp_avg_sq/transformer.blocks.1.q_norm.weight.norm":7.554110670326963e-10,"optim/exp_avg/transformer.blocks.3.ffn.experts.mlp.w1.max":5.7799366004473995e-06,"optim/exp_avg_sq/transformer.blocks.3.k_norm.weight.avg":8.796349065809395e-12,"optim/param/transformer.blocks.3.attn_out.weight.max":0.12098605185747147,"optim/param/transformer.blocks.10.ffn.experts.mlp.w2.max":0.3716731071472168,"optim/grad/transformer.blocks.9.q_norm.weight.norm":0.0001398590102326125,"optim/grad/transformer.blocks.15.ffn.router.layer.weight.max":0.00015413911023642868,"optim/param/transformer.blocks.11.k_norm.weight.max":1.4650007486343384,"optim/param/transformer.blocks.3.ffn.router.layer.weight.max":0.10277055948972702,"optim/grad/transformer.blocks.13.attn_norm.weight.min":1.382431946694851e-10,"optim/exp_avg/transformer.blocks.2.q_norm.weight.norm":2.493710235285107e-05,"optim/grad/transformer.blocks.3.ffn.experts.mlp.w1.avg":-4.4656240510976275e-11,"optim/exp_avg_sq/transformer.blocks.1.att_proj.weight.min":5.203585975196816e-16,"optim/exp_avg/transformer.blocks.1.k_norm.weight.min":1.7716866862471647e-10,"optim/exp_avg_sq/transformer.blocks.11.ffn.experts.mlp.w1.min":5.607143176213102e-16,"optim/grad/transformer.blocks.15.ffn.router.layer.weight.norm":0.002114236820489168,"optim/grad/transformer.wte.weight.max":0.0001454246521461755,"optim/param/transformer.blocks.9.attn_out.weight.avg":-6.951913746888749e-06,"optim/grad/transformer.blocks.12.ffn.experts.mlp.w1.max":1.192699528473895e-05,"optim/grad/transformer.blocks.10.k_norm.weight.min":2.255546860396862e-10,"optim/grad/transformer.blocks.4.attn_out.weight.avg":-6.69693855925857e-10,"optim/exp_avg/transformer.blocks.14.ff_norm.weight.avg":-2.0749726559188275e-07,"optim/exp_avg_sq/transformer.blocks.2.attn_norm.weight.min":8.926201791603638e-12,"optim/exp_avg/transformer.blocks.0.ffn.experts.mlp.w2.min":2.1388963640666292e-16,"optim/param/transformer.blocks.1.k_norm.weight.norm":40.42166519165039,"optim/exp_avg/transformer.blocks.7.ffn.experts.mlp.w1.norm":0.0006796663510613143,"optim/exp_avg_sq/transformer.blocks.5.ff_norm.weight.norm":4.3806269722779234e-09,"optim/exp_avg/transformer.blocks.3.k_norm.weight.avg":-9.011434087824455e-08,"optim/grad/transformer.blocks.8.q_norm.weight.norm":0.00014607765479013324,"optim/exp_avg_sq/transformer.blocks.7.ffn.experts.mlp.w1.min":8.705947568689548e-16,"optim/exp_avg/transformer.blocks.3.attn_norm.weight.min":4.054607760650697e-10,"optim/param/transformer.blocks.3.k_norm.weight.min":0.5140004754066467,"optim/exp_avg_sq/transformer.blocks.6.ffn.experts.mlp.w1.norm":3.798473979799155e-09,"optim/param/transformer.blocks.0.ffn.experts.mlp.w2.avg":-1.9879969840985723e-05,"optim/param/transformer.blocks.4.ff_norm.weight.avg":0.9296228885650635,"optim/grad/transformer.blocks.8.ffn.experts.mlp.w2.min":0,"optim/grad/transformer.blocks.11.ffn.experts.mlp.w1.avg":3.5048682706095846e-12,"optim/exp_avg_sq/transformer.blocks.3.attn_out.weight.avg":7.159868184447582e-12,"optim/exp_avg/transformer.blocks.13.q_norm.weight.norm":1.7586253306944855e-05,"optim/exp_avg_sq/transformer.blocks.2.ffn.experts.mlp.w1.min":6.122444669400321e-16,"optim/exp_avg/transformer.blocks.5.k_norm.weight.avg":-2.3037911489609542e-07,"optim/exp_avg_sq/transformer.blocks.1.attn_norm.weight.min":7.231666220713917e-12,"optim/param/transformer.blocks.10.ffn.experts.mlp.w1.min":4.7892041027397525e-11,"optim/exp_avg/transformer.blocks.8.attn_out.weight.avg":1.3996910575020394e-10,"optim/exp_avg_sq/transformer.blocks.13.attn_out.weight.min":4.8720128951150465e-14,"optim/param/transformer.blocks.14.q_norm.weight.min":0.5003114342689514,"optim/param/transformer.blocks.10.k_norm.weight.max":1.4701647758483887,"optim/exp_avg/transformer.blocks.12.attn_norm.weight.norm":3.239068610128015e-05,"optim/exp_avg/transformer.blocks.5.q_norm.weight.max":1.2726496606774163e-05,"optim/exp_avg_sq/transformer.blocks.5.q_norm.weight.max":3.769656586882775e-08,"optim/exp_avg/transformer.blocks.6.q_norm.weight.avg":-1.9033971909721004e-07,"optim/exp_avg/transformer.blocks.9.ff_norm.weight.max":3.3336593787680613e-06,"optim/param/transformer.blocks.4.attn_norm.weight.min":0.4982640743255615,"optim/exp_avg_sq/transformer.blocks.5.attn_norm.weight.max":4.7011528891971466e-08,"optim/exp_avg/transformer.blocks.10.attn_norm.weight.max":1.2355367289274e-05,"optim/grad/transformer.blocks.10.k_norm.weight.max":3.6462617572396994e-05,"optim/exp_avg/transformer.blocks.15.ffn.experts.mlp.w1.norm":0.0005398050998337567,"optim/exp_avg_sq/transformer.blocks.10.k_norm.weight.max":4.030018541101299e-10,"optim/exp_avg/transformer.blocks.9.q_norm.weight.avg":-1.7385498551902856e-07,"optim/exp_avg/transformer.blocks.12.ffn.experts.mlp.w2.max":4.48779303496849e-07,"optim/exp_avg/transformer.blocks.11.ff_norm.weight.avg":-1.4961314320771635e-07,"optim/param/transformer.blocks.10.ffn.experts.mlp.w1.norm":328.43212890625,"optim/param/transformer.blocks.3.k_norm.weight.max":1.3634634017944336,"optim/exp_avg_sq/transformer.blocks.5.ffn.experts.mlp.w2.norm":7.761556952701198e-10,"optim/exp_avg_sq/transformer.blocks.1.ffn.experts.mlp.w1.min":1.6069053770555206e-17,"optim/exp_avg/transformer.blocks.5.attn_norm.weight.norm":8.970862836576998e-05,"optim/exp_avg_sq/transformer.blocks.1.q_norm.weight.max":3.648698565505981e-10,"optim/exp_avg_sq/transformer.blocks.0.attn_out.weight.max":3.0670308515112765e-09,"optim/grad/transformer.blocks.4.ffn.router.layer.weight.avg":2.6853008705529646e-10,"optim/exp_avg_sq/transformer.blocks.7.ffn.experts.mlp.w1.norm":1.2519827841117603e-08,"optim/exp_avg_sq/transformer.blocks.1.k_norm.weight.max":3.4215313915453294e-10,"optim/param/transformer.blocks.3.attn_norm.weight.avg":0.48481884598731995,"optim/grad/transformer.blocks.12.attn_norm.weight.min":5.002220859751105e-10,"optim/exp_avg_sq/transformer.blocks.10.attn_out.weight.avg":2.486941208523774e-12,"eval/downstream/piqa_len_norm":0.5788900852203369,"optim/param/transformer.blocks.0.attn_out.weight.norm":31.594032287597656,"optim/exp_avg_sq/transformer.blocks.12.attn_out.weight.avg":1.2086851701800794e-12,"optim/exp_avg_sq/transformer.blocks.10.k_norm.weight.norm":1.2136410765251071e-09,"optim/param/transformer.blocks.9.attn_norm.weight.max":0.7569702863693237,"optim/exp_avg/transformer.blocks.0.attn_out.weight.norm":0.001247858046554029,"optim/exp_avg_sq/transformer.blocks.0.k_norm.weight.avg":3.5258345722211093e-12,"optim/exp_avg_sq/transformer.blocks.13.ffn.experts.mlp.w2.avg":2.5725571372041574e-14,"optim/grad/transformer.blocks.1.ff_norm.weight.min":4.310095391701907e-09,"optim/grad/transformer.blocks.3.attn_norm.weight.avg":7.066039415803971e-07,"optim/param/transformer.blocks.4.ffn.experts.mlp.w1.avg":5.309729021973908e-05,"optim/param/transformer.blocks.0.ffn.experts.mlp.w2.min":3.0417052210296447e-10,"optim/exp_avg/transformer.blocks.15.attn_norm.weight.max":6.1383284446492326e-06,"optim/exp_avg_sq/transformer.blocks.15.ffn.experts.mlp.w1.min":1.248187354027243e-16,"optim/exp_avg_sq/transformer.blocks.15.attn_out.weight.max":1.8668995516168962e-10,"optim/param/transformer.blocks.0.ffn.router.layer.weight.norm":4.429228782653809,"optim/exp_avg_sq/transformer.blocks.14.att_proj.weight.avg":9.135419200625994e-13,"optim/exp_avg_sq/transformer.blocks.10.attn_norm.weight.avg":2.2124174020587972e-11,"optim/exp_avg_sq/transformer.blocks.4.ffn.router.layer.weight.norm":1.1010983058667989e-07,"optim/exp_avg/transformer.blocks.1.q_norm.weight.max":3.851539077004418e-06,"optim/param/transformer.blocks.5.attn_out.weight.min":1.1339501782003936e-08,"optim/exp_avg_sq/transformer.blocks.8.ffn.experts.mlp.w1.norm":4.718398116665412e-09,"optim/param/transformer.blocks.6.att_proj.weight.norm":79.27892303466797,"optim/exp_avg_sq/transformer.blocks.10.att_proj.weight.max":5.935844704652027e-09,"optim/param/transformer.blocks.10.ff_norm.weight.avg":1.0839875936508179,"optim/exp_avg/transformer.blocks.5.ff_norm.weight.min":5.722376461569922e-10,"optim/param/transformer.blocks.5.k_norm.weight.max":1.393503189086914,"optim/param/transformer.blocks.11.att_proj.weight.max":0.15843863785266876,"optim/exp_avg_sq/transformer.blocks.10.ffn.experts.mlp.w2.max":1.4798171368846091e-12,"optim/param/transformer.blocks.6.attn_norm.weight.max":0.6895665526390076,"optim/grad/transformer.blocks.2.ffn.experts.mlp.w1.max":4.399266617838293e-05,"optim/exp_avg_sq/transformer.blocks.12.att_proj.weight.max":2.4544200005749417e-09,"optim/exp_avg_sq/transformer.blocks.6.ff_norm.weight.max":3.5582061741479265e-09,"optim/exp_avg_sq/transformer.blocks.4.att_proj.weight.max":3.1820962931305985e-07,"optim/grad/transformer.blocks.15.k_norm.weight.avg":-5.307657602315885e-07,"optim/grad/transformer.blocks.2.attn_out.weight.max":7.982312672538683e-05,"optim/exp_avg_sq/transformer.blocks.5.ffn.experts.mlp.w1.max":3.8705266769412106e-10,"optim/exp_avg_sq/transformer.blocks.13.ffn.experts.mlp.w1.avg":4.358461562579269e-14,"optim/exp_avg/transformer.blocks.14.attn_out.weight.norm":0.0004597949155140668,"optim/exp_avg_sq/transformer.blocks.14.ff_norm.weight.min":1.4813370799102166e-12,"optim/param/transformer.blocks.3.attn_out.weight.min":2.217469541321293e-09,"optim/param/transformer.blocks.15.ffn.experts.mlp.w2.min":4.73875522155609e-11,"optim/exp_avg/transformer.blocks.2.ff_norm.weight.min":2.807793442194395e-10,"optim/param/transformer.blocks.6.ffn.experts.mlp.w1.min":1.850145384119095e-11,"optim/grad/transformer.blocks.11.k_norm.weight.max":3.0229679396143183e-05,"optim/exp_avg_sq/transformer.blocks.15.k_norm.weight.min":1.6503584650346867e-15,"optim/exp_avg_sq/transformer.blocks.5.ffn.experts.mlp.w2.min":5.41913891961409e-16,"optim/exp_avg_sq/transformer.blocks.6.attn_norm.weight.max":2.4820044686180154e-08,"optim/grad/transformer.blocks.4.attn_norm.weight.max":0.00037241028621792793,"optim/exp_avg/transformer.blocks.5.ffn.experts.mlp.w2.max":9.55888594944554e-07,"optim/grad/transformer.blocks.11.ffn.experts.mlp.w2.max":2.4654545995872468e-06,"optim/exp_avg_sq/transformer.blocks.7.attn_norm.weight.max":2.592156533864909e-08,"optim/grad/transformer.blocks.3.att_proj.weight.avg":-2.3820401207075292e-09,"optim/grad/transformer.blocks.5.att_proj.weight.avg":-9.42879441190314e-10,"optim/exp_avg/transformer.blocks.5.ffn.experts.mlp.w1.max":6.381765615515178e-06,"optim/exp_avg/transformer.blocks.10.q_norm.weight.norm":2.454916830174625e-05,"optim/param/transformer.blocks.9.k_norm.weight.max":1.8987092971801758,"optim/grad/transformer.blocks.10.ff_norm.weight.max":1.4774359442526475e-05,"optim/exp_avg_sq/transformer.wte.weight.max":1.1463245463971816e-08,"optim/param/transformer.blocks.12.ffn.experts.mlp.w2.max":0.35163748264312744,"optim/param/transformer.blocks.4.ff_norm.weight.norm":42.13113784790039,"optim/exp_avg_sq/transformer.blocks.4.ffn.experts.mlp.w2.min":5.14809261165745e-16,"optim/param/transformer.blocks.14.ffn.experts.mlp.w1.max":0.519766628742218,"optim/exp_avg_sq/transformer.blocks.13.ffn.router.layer.weight.avg":5.258248697570522e-11,"optim/exp_avg/transformer.blocks.1.k_norm.weight.avg":-1.2759913659010635e-07,"optim/grad/transformer.blocks.8.ffn.experts.mlp.w1.norm":0.002983302576467395,"optim/exp_avg/transformer.blocks.10.ff_norm.weight.min":5.821951254425528e-10,"optim/exp_avg/transformer.blocks.7.ffn.router.layer.weight.max":3.845422179438174e-05,"optim/param/transformer.blocks.2.ffn.experts.mlp.w1.max":0.1959066540002823,"optim/param/transformer.blocks.2.att_proj.weight.avg":5.581583081948338e-06,"optim/exp_avg/transformer.blocks.4.ffn.router.layer.weight.avg":3.283009974985873e-10,"optim/param/transformer.blocks.6.ffn.router.layer.weight.min":2.3282524352907785e-08,"optim/grad/transformer.blocks.6.q_norm.weight.min":1.7917045624926686e-10,"train/LoadBalancingLoss":0.010791384615004063,"optim/exp_avg_sq/transformer.blocks.4.ffn.experts.mlp.w1.max":4.96415408779427e-10,"optim/grad/transformer.blocks.13.att_proj.weight.max":7.869293040130287e-05,"optim/exp_avg_sq/transformer.blocks.11.attn_out.weight.norm":4.8195816226837e-09,"optim/param/transformer.blocks.6.att_proj.weight.avg":6.799848051741719e-06,"optim/exp_avg_sq/transformer.blocks.12.q_norm.weight.norm":9.316565297012858e-10,"optim/exp_avg/transformer.blocks.0.ff_norm.weight.max":1.692531441221945e-05,"optim/exp_avg/transformer.blocks.3.k_norm.weight.min":1.2661448278716847e-10,"optim/exp_avg/transformer.blocks.6.attn_norm.weight.max":2.6917810828308575e-05,"optim/grad/transformer.blocks.11.q_norm.weight.norm":0.000101499717857223,"optim/grad/transformer.blocks.11.attn_norm.weight.avg":2.856800733752607e-07,"optim/grad/transformer.blocks.7.q_norm.weight.min":2.5337953957205173e-11,"optim/param/transformer.blocks.5.ffn.router.layer.weight.min":1.8878247942666349e-07,"optim/param/transformer.blocks.6.q_norm.weight.min":0.38300248980522156,"optim/grad/transformer.blocks.13.ffn.experts.mlp.w1.max":1.710209107841365e-05,"optim/exp_avg_sq/transformer.blocks.13.attn_norm.weight.norm":1.898145196577161e-09,"optim/exp_avg/transformer.blocks.4.ffn.experts.mlp.w2.min":3.6489787614977153e-16,"eval/downstream/mmlu_other_var_len_norm":0.30140843987464905,"optim/exp_avg_sq/transformer.blocks.11.q_norm.weight.norm":7.814468516720297e-10,"optim/grad/transformer.blocks.3.ff_norm.weight.max":4.6266475692391396e-05,"optim/param/transformer.blocks.14.ffn.experts.mlp.w1.norm":331.7171936035156,"optim/grad/transformer.blocks.4.ffn.experts.mlp.w1.max":4.26984770456329e-05,"optim/grad/transformer.blocks.5.k_norm.weight.avg":2.3651114133826923e-07,"optim/param/transformer.blocks.5.ff_norm.weight.max":1.0647062063217163,"optim/exp_avg_sq/transformer.blocks.11.attn_norm.weight.norm":3.2418594475558393e-09,"optim/param/transformer.blocks.13.k_norm.weight.min":0.41200119256973267,"optim/exp_avg_sq/transformer.blocks.6.q_norm.weight.avg":1.28092085549536e-11,"optim/exp_avg_sq/transformer.blocks.13.att_proj.weight.min":8.773476211618213e-16,"optim/param/transformer.blocks.2.q_norm.weight.min":0.5971378087997437,"optim/grad/transformer.blocks.4.att_proj.weight.min":0,"optim/grad/transformer.blocks.14.q_norm.weight.avg":-1.1722307391437425e-07,"optim/grad/transformer.blocks.7.ffn.router.layer.weight.avg":-5.731592978008848e-11,"optim/param/transformer.blocks.7.attn_norm.weight.avg":0.6861681342124939,"optim/grad/transformer.blocks.8.ffn.router.layer.weight.avg":9.394796052220045e-11,"optim/exp_avg/transformer.blocks.0.ffn.router.layer.weight.min":8.500911780062737e-12,"optim/param/transformer.blocks.4.ffn.experts.mlp.w2.max":0.35273587703704834,"optim/param/transformer.blocks.7.k_norm.weight.max":1.4786211252212524,"optim/exp_avg/transformer.blocks.8.attn_norm.weight.max":2.4405189833487384e-05,"optim/exp_avg_sq/transformer.blocks.12.ffn.router.layer.weight.min":6.691003653913619e-12,"optim/param/transformer.blocks.10.ffn.experts.mlp.w1.avg":3.201136860297993e-05,"optim/param/transformer.blocks.4.ffn.router.layer.weight.max":0.11925403028726578,"optim/exp_avg_sq/transformer.blocks.10.ffn.experts.mlp.w2.min":4.0660972246973394e-16,"optim/param/transformer.blocks.10.attn_norm.weight.max":0.7848931550979614,"optim/param/transformer.blocks.0.attn_norm.weight.min":0.24534980952739716,"optim/exp_avg_sq/transformer.blocks.3.attn_out.weight.min":5.174329931537103e-13,"optim/grad/transformer.blocks.3.ffn.experts.mlp.w1.max":3.679533983813599e-05,"optim/exp_avg_sq/transformer.ln_f.weight.norm":3.327904707361995e-08,"optim/param/transformer.blocks.0.q_norm.weight.min":0.6158479452133179,"optim/grad/transformer.blocks.12.attn_out.weight.norm":0.0022525021340698004,"optim/exp_avg/transformer.blocks.2.k_norm.weight.norm":2.5193341571139172e-05,"optim/param/transformer.blocks.15.ffn.experts.mlp.w1.max":0.197786346077919,"optim/exp_avg_sq/transformer.blocks.14.ffn.router.layer.weight.max":1.3048412483840366e-07,"optim/param/transformer.blocks.1.q_norm.weight.max":1.272975206375122,"optim/grad/transformer.blocks.3.attn_norm.weight.max":0.0002106023021042347,"optim/exp_avg/transformer.blocks.11.ffn.experts.mlp.w2.norm":0.00047004339285194874,"optim/grad/transformer.blocks.6.k_norm.weight.max":5.490839612320997e-05,"optim/exp_avg_sq/transformer.blocks.7.ff_norm.weight.max":8.904029180634154e-10,"optim/grad/transformer.blocks.5.q_norm.weight.avg":2.318795395694906e-07,"optim/param/transformer.blocks.2.attn_norm.weight.max":0.6207315325737,"optim/grad/transformer.blocks.11.ffn.router.layer.weight.max":0.00014753587311133742,"optim/param/transformer.blocks.4.k_norm.weight.max":1.6693633794784546,"optim/param/transformer.blocks.9.q_norm.weight.norm":43.2786979675293,"optim/exp_avg/transformer.blocks.0.attn_out.weight.max":1.7912945622811094e-05,"optim/exp_avg/transformer.ff_out.weight.avg":-6.580187241443658e-13,"optim/exp_avg/transformer.blocks.1.q_norm.weight.avg":-1.3070146565041796e-07,"optim/param/transformer.blocks.8.q_norm.weight.avg":0.9666274189949036,"optim/param/transformer.blocks.8.k_norm.weight.norm":43.725830078125,"optim/grad/transformer.blocks.2.att_proj.weight.norm":0.019634446129202843,"optim/exp_avg_sq/transformer.blocks.0.attn_norm.weight.norm":1.3656479502799357e-08,"optim/param/transformer.blocks.10.ff_norm.weight.max":1.170125961303711,"optim/param/transformer.blocks.15.ffn.router.layer.weight.avg":3.108635428361595e-05,"optim/exp_avg_sq/transformer.blocks.9.att_proj.weight.norm":2.8223939807503484e-07,"optim/exp_avg/transformer.blocks.2.ffn.router.layer.weight.max":4.850130062550306e-05,"optim/param/transformer.blocks.8.attn_norm.weight.avg":0.6581895351409912,"optim/param/transformer.blocks.14.ffn.experts.mlp.w2.min":9.735932893217836e-11,"optim/exp_avg/transformer.blocks.1.ffn.experts.mlp.w1.avg":1.3423326686301529e-11,"optim/param/transformer.blocks.9.ffn.router.layer.weight.avg":0.00025137647753581405,"optim/exp_avg/transformer.blocks.14.att_proj.weight.norm":0.0006964736967347562,"optim/grad/transformer.blocks.3.ffn.experts.mlp.w1.norm":0.0031522787176072598,"optim/exp_avg/transformer.blocks.0.att_proj.weight.avg":-2.7145929601402052e-11,"optim/grad/transformer.blocks.2.attn_out.weight.avg":-1.1554435186411638e-09,"optim/exp_avg_sq/transformer.blocks.9.ff_norm.weight.min":1.5573169923760455e-12,"optim/exp_avg_sq/transformer.blocks.0.attn_norm.weight.avg":4.649108181764561e-11,"optim/exp_avg/transformer.blocks.5.k_norm.weight.max":1.1504720532684587e-05,"optim/exp_avg_sq/transformer.blocks.15.att_proj.weight.norm":1.0532239258509435e-07,"optim/grad/transformer.blocks.13.attn_out.weight.avg":3.0017188734632327e-10,"optim/grad/transformer.blocks.8.ffn.experts.mlp.w1.min":0,"optim/grad/transformer.blocks.1.q_norm.weight.norm":0.00010103822569362819,"optim/param/transformer.blocks.14.ffn.router.layer.weight.norm":7.896897792816162,"optim/grad/transformer.blocks.0.ffn.experts.mlp.w2.max":1.0702262443373911e-05,"optim/grad/transformer.blocks.11.ffn.experts.mlp.w1.norm":0.0025443711783736944,"optim/exp_avg/transformer.blocks.8.q_norm.weight.norm":2.9683118555112742e-05,"optim/grad/transformer.blocks.12.k_norm.weight.avg":-3.5986121815767547e-07,"optim/exp_avg_sq/transformer.blocks.3.ff_norm.weight.avg":9.825211824687763e-12,"optim/exp_avg/transformer.blocks.13.attn_out.weight.avg":-7.377572858180415e-11,"optim/param/transformer.blocks.7.ff_norm.weight.min":0.5602085590362549,"optim/exp_avg/transformer.blocks.6.q_norm.weight.max":6.524694526888197e-06,"optim/exp_avg/transformer.blocks.5.q_norm.weight.avg":-2.274585853001554e-07,"optim/exp_avg_sq/transformer.blocks.14.k_norm.weight.avg":2.4154129217579534e-12,"optim/param/transformer.blocks.4.attn_out.weight.norm":36.085716247558594,"optim/param/transformer.blocks.0.ffn.experts.mlp.w2.max":0.40224963426589966,"optim/grad/transformer.blocks.10.q_norm.weight.max":2.983440936077386e-05,"optim/exp_avg_sq/transformer.blocks.3.att_proj.weight.min":1.2288820702609986e-15,"optim/exp_avg/transformer.blocks.9.q_norm.weight.max":4.757770511787385e-06,"optim/exp_avg/transformer.blocks.8.ffn.router.layer.weight.min":3.239403970761723e-12,"optim/exp_avg/transformer.blocks.0.attn_norm.weight.norm":8.892331970855594e-05,"optim/exp_avg/transformer.blocks.8.att_proj.weight.norm":0.0016509484266862273,"train/CrossEntropyLoss":1.2757253646850586,"optim/grad/transformer.blocks.3.k_norm.weight.max":3.840893623419106e-05,"optim/grad/transformer.blocks.1.q_norm.weight.min":1.2960299500264227e-11,"optim/grad/transformer.blocks.12.q_norm.weight.norm":9.956269786925986e-05,"optim/exp_avg_sq/transformer.blocks.3.ffn.router.layer.weight.max":5.7529788932697556e-08,"optim/grad/transformer.blocks.13.ffn.experts.mlp.w1.norm":0.0024303486570715904,"optim/grad/transformer.blocks.9.q_norm.weight.max":2.179801776946988e-05,"optim/exp_avg/transformer.blocks.14.ffn.experts.mlp.w1.avg":-4.288713481570561e-14,"optim/param/transformer.blocks.2.k_norm.weight.avg":0.9465466737747192,"optim/exp_avg/transformer.blocks.5.attn_out.weight.max":4.655329576053191e-06,"optim/param/transformer.blocks.10.att_proj.weight.avg":6.355849109240808e-07,"optim/exp_avg/transformer.blocks.12.ffn.router.layer.weight.min":1.2111183366955713e-12,"optim/exp_avg/transformer.blocks.0.ffn.experts.mlp.w1.min":9.534368290416915e-17,"optim/param/transformer.blocks.15.ffn.experts.mlp.w2.norm":325.0170593261719,"optim/grad/transformer.blocks.10.attn_out.weight.min":0,"optim/exp_avg/transformer.blocks.6.ffn.experts.mlp.w1.max":8.071056072367355e-06,"optim/grad/transformer.blocks.9.k_norm.weight.max":2.513351137167774e-05,"optim/param/transformer.blocks.7.attn_out.weight.min":2.282552591381659e-09,"optim/exp_avg/transformer.blocks.6.ffn.experts.mlp.w2.max":9.86342001851881e-07,"optim/exp_avg/transformer.blocks.3.ffn.experts.mlp.w2.avg":6.89451620794479e-12,"optim/exp_avg/transformer.blocks.12.ffn.router.layer.weight.max":2.0022842363687232e-05,"optim/param/transformer.blocks.4.ffn.experts.mlp.w2.avg":-2.591229167592246e-05,"optim/grad/transformer.blocks.11.ffn.experts.mlp.w2.min":0,"optim/exp_avg_sq/transformer.blocks.12.q_norm.weight.max":3.486633481930568e-10,"optim/param/transformer.blocks.14.att_proj.weight.max":0.163863405585289,"optim/exp_avg/transformer.blocks.1.att_proj.weight.min":2.691415637923254e-15,"optim/param/transformer.blocks.3.q_norm.weight.max":1.2067779302597046,"optim/param/transformer.blocks.13.ffn.experts.mlp.w2.norm":336.5745544433594,"optim/grad/transformer.blocks.7.ffn.router.layer.weight.norm":0.002890887437388301,"eval/c4_en-validation/CrossEntropyLoss":3.698155403137207,"optim/grad/transformer.blocks.8.attn_norm.weight.norm":0.0002752473810687661,"optim/param/transformer.blocks.12.k_norm.weight.norm":42.465187072753906,"optim/grad/transformer.blocks.0.q_norm.weight.max":1.1955992704315577e-05,"optim/exp_avg_sq/transformer.blocks.4.k_norm.weight.norm":1.415290107331657e-08,"optim/exp_avg/transformer.blocks.4.ff_norm.weight.norm":2.7617525120149367e-05,"optim/exp_avg/transformer.blocks.3.attn_norm.weight.max":4.221898052492179e-05,"optim/param/transformer.blocks.13.ff_norm.weight.min":0.5608668327331543,"optim/exp_avg/transformer.blocks.13.ffn.experts.mlp.w1.max":2.085825144604314e-06,"optim/grad/transformer.blocks.13.ffn.experts.mlp.w1.min":0,"optim/exp_avg/transformer.blocks.2.ffn.experts.mlp.w2.max":2.447481620038161e-06,"optim/exp_avg/transformer.blocks.9.ffn.experts.mlp.w1.norm":0.0006115678697824478,"optim/exp_avg/transformer.blocks.9.ffn.experts.mlp.w2.avg":2.965234655039062e-11,"optim/grad/transformer.blocks.12.q_norm.weight.max":3.251299131079577e-05,"optim/exp_avg/transformer.blocks.2.ffn.experts.mlp.w1.max":7.394671683869092e-06,"optim/grad/transformer.blocks.0.att_proj.weight.min":0,"optim/exp_avg_sq/transformer.blocks.11.attn_out.weight.avg":1.9478672147466014e-12,"optim/exp_avg_sq/transformer.blocks.15.ffn.router.layer.weight.max":1.560034590397663e-08,"optim/grad/transformer.blocks.9.ffn.experts.mlp.w1.max":9.505418347544037e-06,"optim/param/transformer.blocks.4.ffn.experts.mlp.w2.norm":309.66729736328125,"optim/grad/transformer.blocks.4.attn_norm.weight.avg":3.572739899482258e-07,"optim/param/transformer.blocks.10.ffn.router.layer.weight.norm":7.812440872192383,"optim/param/transformer.blocks.2.ffn.experts.mlp.w1.min":2.370676988050491e-10,"optim/grad/transformer.blocks.13.k_norm.weight.max":2.1277814084896818e-05,"optim/exp_avg/transformer.blocks.14.q_norm.weight.norm":1.4088614079810213e-05,"optim/exp_avg/transformer.blocks.6.ffn.router.layer.weight.norm":0.0006360385450534523,"optim/grad/transformer.blocks.11.ff_norm.weight.norm":8.725966472411528e-05,"optim/grad/transformer.blocks.2.q_norm.weight.max":1.8093252947437577e-05,"optim/exp_avg_sq/transformer.blocks.8.q_norm.weight.avg":2.5374528866972668e-11,"optim/exp_avg_sq/transformer.blocks.13.ff_norm.weight.avg":3.278176124651977e-12,"optim/param/transformer.blocks.6.attn_out.weight.max":0.12737677991390228,"optim/exp_avg/transformer.blocks.12.q_norm.weight.min":4.4980182772880184e-11,"optim/param/transformer.blocks.12.attn_out.weight.min":4.590624325118142e-09,"optim/exp_avg_sq/transformer.blocks.8.att_proj.weight.norm":6.462904593718122e-07,"optim/exp_avg/transformer.blocks.4.ffn.router.layer.weight.norm":0.0006776338559575379,"optim/param/transformer.ln_f.weight.max":1.3452017307281494,"optim/param/transformer.blocks.10.ff_norm.weight.norm":49.09383010864258,"optim/exp_avg/transformer.blocks.5.att_proj.weight.avg":4.181873458630747e-10,"optim/exp_avg/transformer.blocks.1.k_norm.weight.max":4.206691301078536e-06,"eval/m2d2_s2orc-validation/Perplexity":23.49042510986328,"optim/grad/transformer.blocks.9.attn_norm.weight.norm":0.00021741983073297888,"optim/exp_avg_sq/transformer.blocks.13.ffn.router.layer.weight.norm":5.455367357853902e-08,"optim/param/transformer.blocks.6.ffn.experts.mlp.w1.max":0.19379794597625732,"_step":13954,"optim/param/transformer.blocks.2.attn_out.weight.max":0.15889452397823334,"optim/exp_avg/transformer.blocks.12.k_norm.weight.max":4.028670900879661e-06,"optim/param/transformer.blocks.0.ffn.experts.mlp.w1.avg":-7.002316124271601e-05,"optim/param/transformer.blocks.4.ffn.experts.mlp.w2.min":5.981894840578761e-11,"optim/exp_avg_sq/transformer.blocks.11.att_proj.weight.max":2.8491267123342823e-09,"optim/param/transformer.blocks.8.ffn.experts.mlp.w1.norm":323.1769714355469,"optim/exp_avg/transformer.blocks.1.ffn.router.layer.weight.min":1.2183010676680706e-11,"optim/grad/transformer.blocks.11.ffn.experts.mlp.w2.avg":-1.3859666192495013e-10,"optim/exp_avg/transformer.blocks.5.ffn.router.layer.weight.min":2.304255918017528e-11,"optim/exp_avg/transformer.blocks.11.attn_out.weight.norm":0.0006416343385353684,"optim/grad/transformer.blocks.4.ffn.experts.mlp.w1.avg":1.8356998213175935e-11,"optim/param/transformer.blocks.9.ffn.experts.mlp.w2.min":6.166995802914244e-10,"optim/exp_avg/transformer.blocks.4.attn_out.weight.norm":0.0011932487832382321,"optim/exp_avg_sq/transformer.blocks.13.k_norm.weight.norm":6.854091183505773e-10,"optim/grad/transformer.blocks.12.q_norm.weight.min":1.3733369996771216e-09,"optim/param/transformer.blocks.9.att_proj.weight.max":0.2030605673789978,"optim/exp_avg/transformer.blocks.5.attn_out.weight.min":4.0861598501508445e-13,"optim/grad/transformer.blocks.14.k_norm.weight.min":2.9120883482391946e-10,"optim/exp_avg_sq/transformer.blocks.1.k_norm.weight.min":4.943939460160798e-15,"optim/exp_avg_sq/transformer.blocks.0.ffn.experts.mlp.w1.norm":2.8714173261334963e-09,"optim/exp_avg_sq/transformer.blocks.9.attn_norm.weight.min":6.7240731212170335e-12,"optim/param/transformer.blocks.15.k_norm.weight.min":0.6006195545196533,"optim/grad/transformer.blocks.10.ffn.experts.mlp.w1.norm":0.0026050314772874117,"optim/exp_avg/transformer.blocks.3.ff_norm.weight.min":1.9247480345652157e-11,"optim/exp_avg/transformer.blocks.14.ffn.experts.mlp.w1.max":4.337258815212408e-06,"optim/param/transformer.blocks.1.ff_norm.weight.min":0.49797773361206055,"optim/grad/transformer.blocks.13.k_norm.weight.min":1.9247181626269594e-10,"optim/exp_avg/transformer.blocks.6.ff_norm.weight.max":1.2028137462039012e-05,"optim/exp_avg_sq/transformer.blocks.8.att_proj.weight.min":1.0264291256477224e-15,"optim/param/transformer.blocks.7.ffn.router.layer.weight.norm":7.39522647857666,"optim/grad/transformer.blocks.4.k_norm.weight.min":4.938556230627e-10,"optim/exp_avg/transformer.blocks.8.ffn.experts.mlp.w2.norm":0.0005543519509956241,"optim/param/transformer.blocks.10.ffn.router.layer.weight.avg":0.00024405296426266432,"optim/exp_avg/transformer.blocks.14.q_norm.weight.max":3.2300304155796766e-06,"optim/param/transformer.blocks.13.att_proj.weight.max":0.16398218274116516,"optim/exp_avg_sq/transformer.blocks.2.att_proj.weight.max":3.832718391549861e-07,"optim/grad/transformer.blocks.4.attn_out.weight.max":7.356447167694569e-05,"optim/grad/transformer.blocks.0.ffn.router.layer.weight.max":0.00026961241383105516,"optim/exp_avg_sq/transformer.blocks.10.ff_norm.weight.avg":3.4060326174062405e-12,"optim/exp_avg/transformer.blocks.5.q_norm.weight.min":7.391507544918241e-11,"optim/param/transformer.blocks.12.k_norm.weight.avg":0.9280519485473633,"optim/exp_avg/transformer.blocks.13.ffn.experts.mlp.w1.avg":-2.3080920855123033e-12,"optim/exp_avg/transformer.blocks.11.ffn.router.layer.weight.norm":0.0005739387706853449,"optim/grad/transformer.blocks.3.ff_norm.weight.norm":0.00011777616600738838,"optim/grad/transformer.blocks.14.k_norm.weight.max":1.4828034181846306e-05,"optim/grad/transformer.blocks.4.q_norm.weight.max":5.181521555641666e-05,"optim/grad/transformer.blocks.6.attn_norm.weight.norm":0.00033136390266008675,"optim/exp_avg/transformer.blocks.0.ffn.experts.mlp.w2.norm":0.00035163373104296625,"optim/grad/transformer.blocks.9.att_proj.weight.norm":0.007933498360216618,"optim/grad/transformer.blocks.2.att_proj.weight.min":0,"optim/param/transformer.blocks.15.attn_out.weight.min":1.0424325846969396e-09,"optim/exp_avg_sq/transformer.blocks.4.ff_norm.weight.min":2.647295577196118e-12,"optim/grad/transformer.ff_out.weight.max":0.0018433708464726806,"optim/exp_avg/transformer.blocks.13.k_norm.weight.max":5.325222900864901e-06,"optim/exp_avg_sq/transformer.blocks.0.ffn.experts.mlp.w1.avg":8.903812135827548e-15,"optim/exp_avg/transformer.blocks.5.attn_out.weight.norm":0.001194764394313097,"optim/param/transformer.blocks.4.att_proj.weight.norm":67.50991821289062,"optim/exp_avg_sq/transformer.blocks.7.ffn.router.layer.weight.norm":5.3839503522112864e-08,"optim/grad/transformer.blocks.8.k_norm.weight.min":3.5061020753346384e-10,"optim/grad/transformer.blocks.12.att_proj.weight.max":0.00010652828495949507,"optim/exp_avg_sq/transformer.blocks.7.ffn.router.layer.weight.min":4.4386200444279655e-12,"optim/param/transformer.blocks.3.q_norm.weight.min":0.48113352060317993,"optim/exp_avg_sq/transformer.blocks.8.attn_out.weight.min":2.355173619062595e-13,"optim/exp_avg_sq/transformer.blocks.13.k_norm.weight.min":5.5574695642960845e-15,"optim/param/transformer.blocks.9.ffn.experts.mlp.w2.avg":-2.5139648641925305e-05,"optim/exp_avg_sq/transformer.blocks.5.att_proj.weight.norm":3.4233692076668376e-06,"optim/grad/transformer.blocks.2.k_norm.weight.max":1.867259561549872e-05,"optim/exp_avg/transformer.blocks.1.q_norm.weight.min":4.903666882272173e-12,"optim/grad/transformer.blocks.14.attn_norm.weight.avg":-3.029227357842501e-08,"optim/exp_avg_sq/transformer.blocks.0.ff_norm.weight.max":1.2394735238174803e-09,"optim/exp_avg_sq/transformer.blocks.5.q_norm.weight.min":1.46499972526401e-14,"optim/grad/transformer.blocks.12.ff_norm.weight.norm":8.00405177869834e-05,"optim/param/transformer.blocks.5.att_proj.weight.avg":2.79760956800601e-06,"optim/exp_avg_sq/transformer.blocks.1.q_norm.weight.min":3.714345352828514e-15,"optim/exp_avg_sq/transformer.blocks.0.ffn.experts.mlp.w2.avg":1.8418880515843478e-14,"optim/param/transformer.blocks.8.ffn.router.layer.weight.max":0.11219990998506546,"optim/param/transformer.blocks.6.ffn.experts.mlp.w1.norm":320.3362731933594,"optim/grad/transformer.blocks.4.ffn.experts.mlp.w2.avg":-2.432439805133413e-10,"optim/exp_avg_sq/transformer.blocks.4.q_norm.weight.norm":1.219895295889728e-08,"optim/exp_avg/transformer.blocks.9.k_norm.weight.norm":2.9098157028784044e-05,"optim/exp_avg_sq/transformer.blocks.0.ffn.experts.mlp.w2.max":3.094362960309738e-11,"optim/param/transformer.blocks.8.attn_out.weight.avg":1.689268901827745e-05,"optim/exp_avg/transformer.blocks.14.attn_norm.weight.max":6.681382728856988e-06,"optim/exp_avg_sq/transformer.blocks.7.k_norm.weight.avg":1.5418791243781982e-11,"optim/grad/transformer.blocks.10.ff_norm.weight.avg":2.264581695499146e-07,"optim/param/transformer.blocks.2.attn_norm.weight.norm":20.517696380615234,"optim/exp_avg_sq/transformer.blocks.1.ff_norm.weight.max":2.07416972486385e-09,"optim/grad/transformer.blocks.13.ffn.experts.mlp.w2.max":3.782667135965312e-06,"optim/exp_avg/transformer.blocks.10.att_proj.weight.avg":-5.0213191243875244e-11,"optim/exp_avg_sq/transformer.blocks.7.ffn.experts.mlp.w1.max":6.417622433474435e-09,"optim/exp_avg/transformer.blocks.5.attn_norm.weight.max":3.168046896462329e-05,"optim/exp_avg/transformer.blocks.2.ffn.router.layer.weight.min":4.20850550589269e-12,"optim/exp_avg_sq/transformer.blocks.14.ffn.router.layer.weight.norm":1.9640610560145433e-07,"optim/grad/transformer.blocks.8.att_proj.weight.avg":1.0610841094660373e-09,"optim/exp_avg_sq/transformer.blocks.2.att_proj.weight.norm":3.97479652747279e-06,"optim/grad/transformer.blocks.10.ff_norm.weight.norm":8.612927194917575e-05,"optim/exp_avg_sq/transformer.blocks.7.ffn.experts.mlp.w2.min":9.635788574449797e-16,"optim/exp_avg/transformer.blocks.11.att_proj.weight.avg":1.7831373877452705e-10,"optim/grad/transformer.blocks.0.ffn.experts.mlp.w1.avg":-1.7273720648303126e-11,"optim/grad/transformer.blocks.7.q_norm.weight.norm":0.00015981942124199122,"optim/exp_avg_sq/transformer.blocks.12.k_norm.weight.norm":9.479594886840914e-10,"optim/grad/transformer.blocks.5.ffn.experts.mlp.w1.min":0,"optim/param/transformer.blocks.1.ffn.router.layer.weight.avg":-0.00011561973224161193,"optim/grad/transformer.blocks.7.ffn.experts.mlp.w1.min":0,"optim/exp_avg/transformer.blocks.11.k_norm.weight.norm":2.0055547793162987e-05,"optim/grad/transformer.blocks.15.attn_out.weight.norm":0.002484293421730399,"optim/param/transformer.blocks.2.k_norm.weight.min":0.5363312363624573,"optim/exp_avg/transformer.blocks.8.ffn.experts.mlp.w2.min":2.315568378622563e-16,"optim/grad/transformer.blocks.11.ffn.router.layer.weight.min":5.820766091346741e-11,"optim/exp_avg/transformer.blocks.10.k_norm.weight.min":7.665642426379904e-11,"optim/grad/transformer.blocks.14.att_proj.weight.avg":-1.608421208720845e-11,"optim/param/transformer.blocks.1.ff_norm.weight.norm":37.17795181274414,"optim/grad/transformer.blocks.1.attn_norm.weight.max":0.00023744488134980202,"optim/grad/transformer.blocks.14.ffn.router.layer.weight.norm":0.002729590516537428,"optim/param/transformer.blocks.4.ffn.experts.mlp.w1.max":0.17667025327682495,"optim/exp_avg_sq/transformer.blocks.14.ffn.router.layer.weight.avg":5.064252489583865e-11,"optim/exp_avg/transformer.blocks.3.q_norm.weight.min":5.65958252485288e-11,"optim/param/transformer.blocks.6.ffn.experts.mlp.w2.min":6.957728143630604e-11,"optim/exp_avg/transformer.blocks.11.ffn.experts.mlp.w1.avg":-4.594143641900583e-12,"optim/exp_avg_sq/transformer.blocks.0.q_norm.weight.avg":3.2889628520660352e-12,"optim/exp_avg/transformer.blocks.7.ffn.experts.mlp.w2.avg":-7.364966275735796e-12,"optim/exp_avg/transformer.blocks.4.q_norm.weight.avg":-1.8528757550484443e-07,"optim/grad/transformer.blocks.15.att_proj.weight.max":0.00023316332953982055,"optim/exp_avg/transformer.blocks.5.ffn.router.layer.weight.norm":0.0006718678632751107,"optim/exp_avg/transformer.ln_f.weight.min":4.510458229134429e-09,"optim/exp_avg/transformer.blocks.1.ffn.experts.mlp.w2.max":3.2080408800538862e-06,"optim/exp_avg/transformer.blocks.15.ffn.router.layer.weight.max":3.5979868698632345e-05,"optim/exp_avg/transformer.blocks.1.ffn.experts.mlp.w1.max":7.822385668987408e-06,"optim/exp_avg/transformer.blocks.8.ffn.router.layer.weight.max":3.2720796298235655e-05,"optim/exp_avg/transformer.blocks.9.k_norm.weight.max":4.531787908490514e-06,"optim/exp_avg/transformer.blocks.15.k_norm.weight.norm":1.0206075785390567e-05,"optim/param/transformer.blocks.14.ffn.experts.mlp.w2.norm":334.2730712890625,"optim/exp_avg/transformer.blocks.0.attn_norm.weight.min":4.418712062914665e-09,"optim/grad/transformer.blocks.11.ffn.experts.mlp.w1.max":1.2152609997428954e-05,"optim/exp_avg_sq/transformer.blocks.10.att_proj.weight.norm":1.1507520980558184e-07,"eval/downstream/mmlu_humanities_var_len_norm":0.25234419107437134,"optim/exp_avg/transformer.blocks.4.attn_norm.weight.max":2.0148474504821934e-05,"optim/grad/transformer.blocks.2.ffn.router.layer.weight.min":1.7143975128419697e-10,"optim/exp_avg_sq/transformer.blocks.3.ffn.experts.mlp.w1.avg":7.133272379895861e-14,"optim/exp_avg/transformer.blocks.15.ffn.router.layer.weight.min":2.048475278293438e-11,"optim/param/transformer.blocks.6.ffn.experts.mlp.w2.avg":-1.2826768397644628e-05,"optim/param/transformer.blocks.4.ff_norm.weight.max":1.0570894479751587,"optim/exp_avg/transformer.blocks.12.ff_norm.weight.min":1.074002276002517e-10,"optim/exp_avg_sq/transformer.blocks.3.ffn.experts.mlp.w2.avg":7.286478956010714e-14,"optim/param/transformer.blocks.2.attn_norm.weight.avg":0.45256567001342773,"optim/param/transformer.blocks.4.attn_out.weight.max":0.09942261874675751,"optim/exp_avg_sq/transformer.blocks.2.attn_norm.weight.avg":1.537389382466614e-10,"optim/grad/transformer.blocks.0.ffn.experts.mlp.w2.avg":2.851279767845938e-10,"optim/param/transformer.blocks.7.ffn.experts.mlp.w1.norm":322.22149658203125,"eval/dolma_books-validation/CrossEntropyLoss":3.699963092803955,"optim/grad/transformer.blocks.2.attn_out.weight.min":0,"optim/exp_avg_sq/transformer.blocks.9.ffn.experts.mlp.w2.min":1.9797469465603247e-16,"optim/param/transformer.blocks.2.ffn.experts.mlp.w2.max":0.3646237552165985,"optim/param/transformer.blocks.8.att_proj.weight.min":1.1486577244923524e-09,"optim/exp_avg_sq/transformer.blocks.6.ffn.experts.mlp.w2.min":1.3802030239803216e-15,"optim/grad/transformer.ff_out.weight.min":6.938893903907228e-18,"optim/exp_avg/transformer.blocks.14.att_proj.weight.avg":2.4991503658200465e-11,"optim/grad/transformer.blocks.5.ffn.router.layer.weight.avg":1.177813402364336e-10,"optim/exp_avg/transformer.blocks.8.attn_out.weight.min":1.1063521518188402e-13,"optim/grad/transformer.blocks.14.k_norm.weight.avg":-1.1783982500901402e-07,"optim/exp_avg_sq/transformer.blocks.3.ffn.experts.mlp.w2.max":6.95441759734905e-11,"optim/exp_avg_sq/transformer.blocks.11.ffn.experts.mlp.w2.min":2.845280019824945e-16,"optim/exp_avg_sq/transformer.blocks.14.ff_norm.weight.avg":4.656680856890416e-12,"optim/param/transformer.blocks.10.k_norm.weight.avg":0.9536257982254028,"optim/exp_avg/transformer.blocks.13.attn_norm.weight.max":7.809154340066016e-06,"optim/param/transformer.blocks.2.k_norm.weight.norm":43.05442810058594,"optim/param/transformer.blocks.8.attn_norm.weight.max":0.7095671892166138,"optim/exp_avg_sq/transformer.blocks.9.ffn.experts.mlp.w1.max":3.289767069869498e-11,"optim/exp_avg_sq/transformer.blocks.6.ffn.experts.mlp.w1.max":1.1761603913029717e-09,"optim/exp_avg_sq/transformer.blocks.9.attn_norm.weight.avg":3.180878824227129e-11,"eval/downstream/mmlu_social_sciences_mc_5shot_len_norm":0.2640949487686157,"optim/grad/transformer.blocks.13.k_norm.weight.norm":8.22730507934466e-05,"optim/exp_avg/transformer.blocks.14.ffn.router.layer.weight.avg":2.7355451237554007e-11,"optim/exp_avg_sq/transformer.blocks.13.ffn.router.layer.weight.max":2.519958641755693e-08,"optim/grad/transformer.wte.weight.norm":0.004240115638822317,"eval/downstream/social_iqa_len_norm":0.3868986666202545,"optim/exp_avg/transformer.blocks.5.ffn.experts.mlp.w1.avg":1.8279565361378758e-11,"optim/exp_avg/transformer.blocks.7.ffn.router.layer.weight.min":1.9052485283888032e-11,"optim/grad/transformer.blocks.12.ffn.router.layer.weight.norm":0.002388998167589307,"eval/downstream/mmlu_humanities_mc_5shot_len_norm":0.25868725776672363,"optim/exp_avg/transformer.blocks.13.k_norm.weight.norm":1.8491260561859235e-05,"optim/grad/transformer.blocks.9.ffn.experts.mlp.w1.norm":0.0027053968515247107,"optim/exp_avg/transformer.blocks.15.att_proj.weight.avg":6.862766604998072e-11,"optim/param/transformer.blocks.9.ff_norm.weight.min":0.643264889717102,"optim/param/transformer.blocks.6.ffn.experts.mlp.w1.avg":5.517146200872958e-05,"optim/grad/transformer.blocks.14.k_norm.weight.norm":7.350259693339467e-05,"optim/grad/transformer.blocks.0.ffn.router.layer.weight.norm":0.0016712654614821076,"optim/grad/transformer.blocks.15.attn_out.weight.avg":-1.5838751143970597e-10,"optim/exp_avg_sq/transformer.blocks.1.attn_out.weight.norm":3.5653293650739215e-08,"optim/grad/transformer.blocks.12.att_proj.weight.min":0,"optim/grad/transformer.blocks.14.ffn.experts.mlp.w2.norm":0.0017277023289352655,"optim/param/transformer.blocks.7.attn_norm.weight.max":0.7423169612884521,"optim/exp_avg_sq/transformer.blocks.13.ffn.experts.mlp.w2.norm":4.0999956207876664e-10,"optim/param/transformer.blocks.0.ffn.experts.mlp.w1.max":0.1761859953403473,"optim/param/transformer.blocks.6.ffn.router.layer.weight.max":0.11195893585681915,"optim/param/transformer.blocks.13.q_norm.weight.min":0.3090692460536957,"optim/exp_avg_sq/transformer.blocks.2.q_norm.weight.max":3.906583945223474e-10,"optim/param/transformer.blocks.14.ffn.router.layer.weight.avg":4.315405385568738e-05,"optim/exp_avg/transformer.blocks.0.att_proj.weight.min":9.814999358506889e-15,"optim/grad/transformer.blocks.7.attn_norm.weight.min":1.8826540326699615e-09,"optim/exp_avg/transformer.blocks.15.ffn.experts.mlp.w1.avg":3.6106003169916745e-12,"optim/exp_avg_sq/transformer.blocks.5.att_proj.weight.max":4.206815731322422e-07,"optim/param/transformer.blocks.15.att_proj.weight.max":0.138942688703537,"optim/grad/transformer.blocks.0.ff_norm.weight.norm":7.234183431137353e-05,"optim/param/transformer.blocks.5.ffn.router.layer.weight.max":0.11552528291940689,"optim/grad/transformer.blocks.1.ff_norm.weight.avg":-2.2334673133173055e-07,"optim/exp_avg/transformer.ff_out.weight.min":1.5781570457384852e-17,"eval/downstream/mmlu_social_sciences_var_len_norm":0.23993217945098877,"optim/exp_avg/transformer.blocks.14.ff_norm.weight.norm":2.169945219065994e-05,"optim/exp_avg_sq/transformer.blocks.9.ffn.router.layer.weight.max":5.5568203372047265e-09,"optim/param/transformer.blocks.1.attn_out.weight.min":1.5821826071515943e-09,"optim/exp_avg/transformer.blocks.11.ff_norm.weight.max":2.427226718282327e-06,"optim/param/transformer.blocks.5.ffn.experts.mlp.w1.norm":319.51483154296875,"optim/exp_avg/transformer.blocks.0.ffn.router.layer.weight.max":7.275473763002083e-05,"optim/grad/transformer.blocks.9.attn_norm.weight.avg":6.663768203907239e-08,"optim/exp_avg_sq/transformer.blocks.0.att_proj.weight.norm":4.130237982735707e-07,"optim/exp_avg_sq/transformer.ln_f.weight.avg":1.004400798643168e-10,"optim/param/transformer.blocks.13.attn_out.weight.min":6.86383705428284e-09,"optim/exp_avg/transformer.blocks.3.k_norm.weight.norm":2.0306646547396667e-05,"optim/exp_avg/transformer.blocks.7.attn_norm.weight.avg":2.066223316887772e-07,"optim/exp_avg/transformer.blocks.12.attn_out.weight.max":1.8298171653441386e-06,"optim/grad/transformer.blocks.4.att_proj.weight.max":0.0008001414826139808,"optim/exp_avg/transformer.blocks.7.q_norm.weight.max":1.338762649538694e-05,"optim/exp_avg_sq/transformer.blocks.14.ffn.experts.mlp.w2.max":5.355603113765817e-12,"optim/exp_avg/transformer.blocks.11.k_norm.weight.min":2.9933482775801323e-11,"eval/downstream/openbook_qa_len_norm":0.24799999594688416,"optim/grad/transformer.blocks.15.k_norm.weight.min":6.752571835022536e-10,"optim/param/transformer.blocks.9.att_proj.weight.min":8.600328227359455e-10,"optim/exp_avg/transformer.blocks.8.att_proj.weight.max":4.3956777517450973e-05,"optim/exp_avg/transformer.blocks.0.ff_norm.weight.avg":2.802303455951005e-08,"optim/param/transformer.blocks.4.ffn.router.layer.weight.min":4.943950315805523e-08,"optim/grad/transformer.blocks.11.attn_norm.weight.max":4.850748518947512e-05,"optim/exp_avg_sq/transformer.blocks.6.k_norm.weight.max":2.5355222366130192e-09,"optim/exp_avg_sq/transformer.ln_f.weight.min":2.2549417194595023e-11,"optim/param/transformer.blocks.10.attn_norm.weight.avg":0.7291692495346069,"optim/param/transformer.blocks.3.attn_norm.weight.max":0.6240735650062561,"optim/exp_avg_sq/transformer.blocks.13.ff_norm.weight.min":1.1848807525416394e-12,"optim/param/transformer.blocks.2.ffn.experts.mlp.w2.avg":-1.1124177035526372e-05,"optim/grad/transformer.blocks.13.ffn.experts.mlp.w2.min":0,"optim/exp_avg/transformer.blocks.9.k_norm.weight.avg":-1.743379414165247e-07,"optim/param/transformer.blocks.15.ffn.experts.mlp.w1.norm":328.4910583496094,"optim/param/transformer.blocks.9.attn_norm.weight.avg":0.6988779306411743,"optim/param/transformer.blocks.2.ffn.experts.mlp.w1.norm":318.1918029785156,"optim/exp_avg/transformer.blocks.5.att_proj.weight.min":1.4267705872555818e-14,"optim/exp_avg_sq/transformer.blocks.9.ff_norm.weight.avg":4.269713055338187e-12,"optim/exp_avg_sq/transformer.blocks.6.ffn.router.layer.weight.norm":6.805520058605907e-08,"optim/grad/transformer.blocks.1.q_norm.weight.avg":1.5312232903852419e-07,"optim/exp_avg_sq/transformer.blocks.13.ffn.experts.mlp.w1.max":5.455118995412178e-11,"optim/exp_avg_sq/transformer.blocks.11.ffn.router.layer.weight.min":7.915090458054941e-12,"optim/exp_avg_sq/transformer.blocks.1.attn_out.weight.max":1.382746916966937e-09,"optim/exp_avg/transformer.blocks.15.att_proj.weight.max":2.4810673494357616e-05,"optim/exp_avg_sq/transformer.blocks.8.ffn.experts.mlp.w1.avg":6.522039175003644e-14,"optim/exp_avg/transformer.blocks.5.ffn.experts.mlp.w2.norm":0.000620369566604495,"optim/param/transformer.blocks.6.k_norm.weight.max":1.324054479598999,"optim/grad/transformer.blocks.15.ffn.router.layer.weight.avg":3.5864022862597267e-10,"optim/param/transformer.blocks.0.att_proj.weight.max":0.2738201320171356,"throughput/total_tokens":58527318016,"optim/exp_avg_sq/transformer.blocks.8.ffn.experts.mlp.w2.avg":4.4318173263169225e-14,"eval/dolma_pes2o-validation/CrossEntropyLoss":2.4730865955352783,"optim/exp_avg/transformer.blocks.12.ffn.experts.mlp.w2.norm":0.00042930679046548903,"optim/grad/transformer.blocks.11.ffn.experts.mlp.w1.min":0,"optim/param/transformer.blocks.10.att_proj.weight.min":1.3237464457915848e-09,"optim/param/transformer.blocks.3.ffn.router.layer.weight.avg":4.995630297344178e-05,"optim/exp_avg_sq/transformer.blocks.8.ffn.experts.mlp.w2.max":7.7712575957678e-12,"optim/param/transformer.blocks.13.ffn.experts.mlp.w1.norm":333.22125244140625,"optim/param/transformer.blocks.6.ffn.experts.mlp.w2.max":0.39802828431129456,"optim/exp_avg_sq/transformer.blocks.2.k_norm.weight.norm":1.2065604071409552e-09,"optim/param/transformer.blocks.9.attn_norm.weight.norm":31.637771606445312,"optim/param/transformer.blocks.6.attn_norm.weight.avg":0.6260622143745422,"optim/exp_avg_sq/transformer.blocks.2.k_norm.weight.min":1.516289772505921e-14,"optim/exp_avg/transformer.blocks.1.attn_out.weight.max":1.0786081475089304e-05,"optim/exp_avg_sq/transformer.blocks.14.k_norm.weight.min":7.954220121682341e-16,"optim/exp_avg_sq/transformer.blocks.3.ff_norm.weight.min":2.3989918124939225e-12,"optim/param/transformer.blocks.4.att_proj.weight.avg":-3.288384732513805e-06,"optim/grad/transformer.blocks.5.ffn.experts.mlp.w2.min":0,"optim/exp_avg/transformer.blocks.9.ffn.router.layer.weight.avg":-3.022937455909869e-11,"optim/grad/transformer.blocks.2.ffn.router.layer.weight.max":0.0002888828166760504,"optim/exp_avg/transformer.blocks.4.q_norm.weight.norm":4.7135403292486444e-05,"optim/grad/transformer.blocks.7.ffn.experts.mlp.w2.norm":0.0025413157418370247,"optim/exp_avg_sq/transformer.blocks.0.ffn.router.layer.weight.min":6.563726884060983e-13,"optim/exp_avg_sq/transformer.blocks.3.q_norm.weight.min":1.4907620629481602e-14,"optim/grad/transformer.blocks.7.ffn.experts.mlp.w2.min":0,"optim/exp_avg_sq/transformer.blocks.6.att_proj.weight.max":1.4768363598705037e-07,"optim/param/transformer.blocks.4.ffn.experts.mlp.w1.min":7.33274135855666e-11,"optim/exp_avg/transformer.blocks.15.attn_out.weight.min":2.2358244406274833e-14,"optim/exp_avg/transformer.blocks.13.ffn.experts.mlp.w1.norm":0.00055231514852494,"optim/grad/transformer.blocks.6.ff_norm.weight.max":4.7474284656345844e-05,"optim/grad/transformer.blocks.2.attn_out.weight.norm":0.0052275280468165874,"optim/grad/transformer.blocks.15.att_proj.weight.norm":0.003712215693667531,"optim/param/transformer.blocks.6.q_norm.weight.avg":0.9662156105041504,"optim/param/transformer.blocks.1.ffn.experts.mlp.w1.norm":313.78515625,"optim/param/transformer.blocks.11.ff_norm.weight.norm":50.47687530517578,"optim/exp_avg_sq/transformer.blocks.0.ff_norm.weight.norm":1.3749656968542467e-09,"optim/exp_avg_sq/transformer.blocks.12.attn_norm.weight.norm":1.9695542974318414e-09,"optim/grad/transformer.blocks.3.q_norm.weight.max":4.235301457811147e-05,"optim/exp_avg_sq/transformer.blocks.11.ff_norm.weight.norm":4.505785189401479e-10,"optim/exp_avg/transformer.blocks.1.att_proj.weight.avg":1.8231294252046837e-10,"optim/grad/transformer.blocks.15.attn_norm.weight.min":5.8107616496272385e-09,"optim/exp_avg_sq/transformer.blocks.4.attn_norm.weight.norm":1.271564542548731e-07,"optim/param/transformer.blocks.8.k_norm.weight.max":1.478699803352356,"optim/exp_avg_sq/transformer.blocks.9.att_proj.weight.max":2.8144452102196738e-08,"optim/exp_avg_sq/transformer.blocks.8.ffn.router.layer.weight.avg":5.635892100230855e-11,"optim/param/transformer.blocks.2.ffn.experts.mlp.w2.min":1.4272656667646544e-10,"optim/param/transformer.blocks.10.attn_out.weight.avg":2.2678425011690706e-05,"optim/exp_avg/transformer.blocks.9.ff_norm.weight.avg":-1.1018218515346234e-07,"optim/exp_avg_sq/transformer.blocks.14.attn_out.weight.norm":2.6393403018687422e-09,"optim/param/transformer.blocks.6.q_norm.weight.norm":44.02855682373047,"optim/exp_avg_sq/transformer.blocks.9.k_norm.weight.norm":5.8792348767156e-09,"optim/grad/transformer.blocks.13.ffn.experts.mlp.w2.avg":4.451995022636268e-11,"optim/exp_avg_sq/transformer.blocks.14.ffn.experts.mlp.w2.min":2.0212866334337577e-17,"optim/grad/transformer.blocks.1.attn_norm.weight.avg":2.2975852687068254e-07,"optim/param/transformer.blocks.14.attn_norm.weight.min":0.526223361492157,"optim/exp_avg_sq/transformer.blocks.3.att_proj.weight.avg":2.293717993318012e-11,"optim/param/transformer.blocks.12.attn_norm.weight.norm":31.001514434814453,"optim/grad/transformer.blocks.14.attn_out.weight.max":1.5117257134988904e-05,"optim/grad/transformer.blocks.5.q_norm.weight.min":6.166374078020453e-10,"optim/param/transformer.blocks.2.q_norm.weight.avg":0.9472681283950806,"optim/param/transformer.blocks.5.attn_norm.weight.norm":26.803861618041992,"optim/exp_avg_sq/transformer.ff_out.weight.avg":3.1427347703394393e-12,"optim/grad/transformer.blocks.14.q_norm.weight.min":1.9417711882852018e-10,"optim/exp_avg/transformer.blocks.4.q_norm.weight.max":9.57641259446973e-06,"optim/exp_avg/transformer.blocks.10.q_norm.weight.min":1.0103656800097838e-10,"optim/param/transformer.blocks.14.k_norm.weight.min":0.5221771001815796,"optim/exp_avg_sq/transformer.blocks.0.attn_out.weight.min":2.648703467927199e-13,"optim/grad/transformer.blocks.12.ff_norm.weight.min":2.687556843739003e-10,"optim/grad/transformer.blocks.6.ff_norm.weight.min":1.7967067833524197e-09,"optim/param/transformer.blocks.2.ff_norm.weight.avg":0.8731173276901245,"optim/exp_avg/transformer.blocks.10.ffn.router.layer.weight.max":2.2103784431237727e-05,"optim/learning_rate_group0":4.000004376396637e-05,"optim/param/transformer.blocks.13.ffn.router.layer.weight.avg":0.00014060217654332519,"optim/param/transformer.blocks.1.attn_out.weight.max":0.1806996911764145,"optim/grad/transformer.blocks.14.ff_norm.weight.avg":-2.366860059055398e-07,"optim/grad/transformer.blocks.3.ff_norm.weight.avg":-5.977049255534439e-08,"optim/exp_avg/transformer.blocks.13.attn_out.weight.norm":0.00048562593292444944,"optim/exp_avg/transformer.blocks.9.ffn.experts.mlp.w1.avg":4.730535407837522e-12,"optim/param/transformer.blocks.8.k_norm.weight.min":0.3229001760482788,"optim/grad/transformer.blocks.3.ffn.experts.mlp.w2.max":1.7064543499145657e-05,"optim/param/transformer.blocks.6.ff_norm.weight.norm":43.0883903503418,"optim/exp_avg/transformer.blocks.3.attn_norm.weight.norm":7.272870425367728e-05,"optim/exp_avg/transformer.blocks.12.ffn.router.layer.weight.norm":0.000554906961042434,"optim/param/transformer.ff_out.weight.max":0.23240093886852264,"optim/exp_avg_sq/transformer.blocks.12.ffn.experts.mlp.w1.avg":4.363770765092659e-14,"optim/exp_avg/transformer.blocks.10.ffn.router.layer.weight.min":9.449457275684248e-13,"optim/grad/transformer.blocks.9.attn_norm.weight.min":4.060439096065238e-09,"optim/param/transformer.blocks.3.attn_norm.weight.norm":21.9719181060791,"optim/exp_avg/transformer.blocks.12.attn_norm.weight.min":3.8239389432703774e-10,"optim/grad/transformer.blocks.11.att_proj.weight.avg":3.858209585150263e-10,"optim/exp_avg/transformer.blocks.8.attn_out.weight.norm":0.0009988048113882542,"_wandb":{"runtime":556201},"optim/exp_avg/transformer.blocks.7.ff_norm.weight.min":1.615438754987153e-10,"optim/param/transformer.blocks.7.k_norm.weight.avg":0.8797732591629028,"optim/exp_avg_sq/transformer.blocks.5.ff_norm.weight.avg":1.0362273539232802e-11,"optim/exp_avg_sq/transformer.blocks.14.ff_norm.weight.norm":1.0226957058989683e-09,"optim/grad/transformer.blocks.15.ffn.experts.mlp.w2.norm":0.00159652775619179,"optim/exp_avg_sq/transformer.blocks.8.attn_out.weight.norm":1.2754411748971961e-08,"optim/exp_avg_sq/transformer.blocks.4.ffn.router.layer.weight.max":2.9852870397917286e-08,"optim/exp_avg/transformer.blocks.1.attn_out.weight.min":1.8461283662809386e-13,"optim/param/transformer.blocks.0.ff_norm.weight.max":0.9694640636444092,"optim/param/transformer.blocks.9.ffn.experts.mlp.w1.min":8.820837116285674e-11,"optim/exp_avg_sq/transformer.blocks.12.attn_out.weight.norm":2.754692252082691e-09,"optim/exp_avg_sq/transformer.blocks.0.q_norm.weight.norm":4.1535999639741306e-10,"optim/param/transformer.blocks.6.ff_norm.weight.max":1.0651134252548218,"optim/exp_avg/transformer.blocks.13.att_proj.weight.min":6.176001821501381e-15,"optim/grad/transformer.blocks.14.q_norm.weight.max":1.3634824426844716e-05,"optim/exp_avg_sq/transformer.blocks.13.ffn.experts.mlp.w1.norm":8.446445209919773e-10,"optim/exp_avg_sq/transformer.blocks.0.attn_norm.weight.min":1.1755433432236728e-11,"optim/grad/transformer.ln_f.weight.avg":-1.6582550870225532e-06,"optim/param/transformer.blocks.6.k_norm.weight.norm":43.74885177612305,"optim/exp_avg/transformer.blocks.3.ffn.experts.mlp.w1.avg":1.0641336770089715e-11,"optim/exp_avg_sq/transformer.blocks.10.ff_norm.weight.min":1.1711022776528246e-12,"optim/param/transformer.blocks.11.att_proj.weight.norm":81.1254653930664,"optim/param/transformer.blocks.9.k_norm.weight.avg":0.9398679733276367,"optim/grad/transformer.blocks.0.ffn.experts.mlp.w1.max":2.7108413632959127e-05,"optim/grad/transformer.blocks.4.attn_out.weight.min":0,"optim/grad/transformer.blocks.14.attn_norm.weight.norm":0.00013269139162730426,"optim/exp_avg/transformer.blocks.0.attn_out.weight.avg":8.30109314620131e-11,"optim/exp_avg_sq/transformer.blocks.15.att_proj.weight.max":3.15144710327786e-08,"optim/exp_avg_sq/transformer.blocks.15.attn_out.weight.norm":6.84076351120666e-09,"optim/exp_avg_sq/transformer.blocks.9.ffn.experts.mlp.w2.norm":4.983564672045304e-10,"optim/param/transformer.blocks.6.k_norm.weight.min":0.3998134136199951,"optim/grad/transformer.blocks.15.ffn.router.layer.weight.min":1.1823431123048067e-11,"optim/exp_avg/transformer.blocks.3.ffn.experts.mlp.w1.norm":0.0006849156343378127,"optim/param/transformer.blocks.9.ffn.experts.mlp.w2.norm":326.4009094238281,"optim/exp_avg_sq/transformer.blocks.10.attn_out.weight.max":2.8320054634711767e-11,"optim/grad/transformer.blocks.11.k_norm.weight.avg":9.686404212061461e-08,"optim/param/transformer.blocks.3.ffn.router.layer.weight.min":9.463722250302453e-08,"eval/downstream/boolq_acc":0.5822629928588867,"optim/param/transformer.blocks.0.ff_norm.weight.norm":31.23689842224121,"eval/ice-validation/Perplexity":24.367753982543945,"optim/exp_avg/transformer.blocks.8.ff_norm.weight.norm":2.3056320060277358e-05,"optim/exp_avg/transformer.blocks.1.attn_norm.weight.min":3.204987109040047e-10,"optim/exp_avg_sq/transformer.blocks.4.q_norm.weight.avg":4.979197956722636e-11,"optim/exp_avg_sq/transformer.blocks.12.attn_out.weight.max":1.2843594243694412e-11,"optim/param/transformer.blocks.11.q_norm.weight.avg":1.0270657539367676,"optim/exp_avg_sq/transformer.blocks.2.ffn.router.layer.weight.max":3.7796443308479866e-08,"optim/grad/transformer.blocks.7.k_norm.weight.norm":0.00016591940948273987,"optim/param/transformer.blocks.12.ffn.router.layer.weight.min":2.4343517424085803e-08,"optim/exp_avg_sq/transformer.blocks.8.ff_norm.weight.min":1.8805649312086947e-12,"optim/exp_avg/transformer.blocks.6.attn_out.weight.norm":0.0011393739841878414,"optim/exp_avg_sq/transformer.blocks.10.ffn.router.layer.weight.avg":4.67496458211869e-11,"optim/param/transformer.blocks.8.ffn.experts.mlp.w1.avg":3.984143404522911e-05,"optim/exp_avg_sq/transformer.blocks.15.attn_out.weight.avg":1.5121543340407273e-12,"optim/param/transformer.blocks.12.ff_norm.weight.max":1.2126333713531494,"optim/exp_avg_sq/transformer.blocks.0.att_proj.weight.min":9.129483100558011e-16,"optim/exp_avg/transformer.blocks.10.ffn.experts.mlp.w1.norm":0.0005915494984947145,"optim/exp_avg_sq/transformer.blocks.9.q_norm.weight.min":2.9312551345206774e-15,"optim/exp_avg/transformer.blocks.9.q_norm.weight.norm":2.841383684426546e-05,"optim/grad/transformer.blocks.7.attn_norm.weight.avg":6.423040588288131e-08,"optim/exp_avg/transformer.blocks.8.ff_norm.weight.max":4.4230910134501755e-06,"optim/param/transformer.blocks.13.attn_norm.weight.avg":0.6820443868637085,"optim/exp_avg_sq/transformer.blocks.8.ff_norm.weight.norm":1.7168894084207409e-09,"optim/param/transformer.blocks.11.ffn.experts.mlp.w2.norm":333.1158752441406,"optim/exp_avg_sq/transformer.blocks.1.ffn.experts.mlp.w1.norm":5.2236259762139525e-09,"optim/param/transformer.blocks.1.ffn.experts.mlp.w2.avg":-7.684937372687273e-06,"optim/exp_avg/transformer.blocks.10.k_norm.weight.norm":2.4527236746507697e-05,"optim/exp_avg/transformer.blocks.11.q_norm.weight.norm":1.9777768102358095e-05,"optim/exp_avg/transformer.blocks.14.attn_out.weight.max":3.2201382964558434e-06,"optim/param/transformer.blocks.4.ffn.router.layer.weight.avg":0.0002962860744446516,"optim/exp_avg/transformer.blocks.14.ff_norm.weight.max":4.425303359312238e-06,"optim/exp_avg_sq/transformer.blocks.8.att_proj.weight.avg":7.417569163059579e-12,"optim/exp_avg_sq/transformer.blocks.4.ff_norm.weight.avg":9.987459644034136e-12,"optim/grad/transformer.blocks.9.ffn.router.layer.weight.avg":-6.357625537134481e-11,"optim/exp_avg_sq/transformer.blocks.12.ff_norm.weight.min":1.11942508214391e-12,"optim/exp_avg/transformer.blocks.0.attn_norm.weight.max":5.1491413614712656e-05,"optim/exp_avg/transformer.blocks.1.ffn.experts.mlp.w2.avg":-4.728754887661779e-11,"optim/exp_avg/transformer.blocks.4.attn_norm.weight.min":3.069407394917789e-09,"optim/param/transformer.blocks.12.att_proj.weight.max":0.1680271476507187,"optim/exp_avg/transformer.blocks.3.ff_norm.weight.norm":2.7427719032857567e-05,"optim/exp_avg_sq/transformer.blocks.13.attn_norm.weight.min":2.3194768588447046e-12,"optim/grad/transformer.blocks.15.k_norm.weight.max":1.8227445252705365e-05,"optim/grad/transformer.blocks.8.q_norm.weight.max":4.6207758714444935e-05,"optim/grad/transformer.blocks.4.attn_norm.weight.norm":0.00029055096092633903,"optim/param/transformer.blocks.5.k_norm.weight.avg":0.9861794710159302,"optim/exp_avg/transformer.blocks.15.ffn.experts.mlp.w1.max":8.163485290424433e-06,"optim/grad/transformer.blocks.12.ffn.experts.mlp.w2.min":0,"optim/exp_avg_sq/transformer.blocks.5.ffn.router.layer.weight.norm":1.0602180822161245e-07,"optim/exp_avg_sq/transformer.blocks.13.attn_norm.weight.max":1.0470037059562287e-09,"optim/grad/transformer.blocks.8.k_norm.weight.avg":-3.3987504366450594e-07,"optim/exp_avg/transformer.ln_f.weight.norm":0.00014512021152768284,"optim/grad/transformer.blocks.14.attn_out.weight.min":0,"optim/exp_avg_sq/transformer.blocks.10.q_norm.weight.norm":1.1896537088773584e-09,"optim/exp_avg/transformer.blocks.3.q_norm.weight.max":4.683819952333579e-06,"optim/exp_avg_sq/transformer.blocks.10.q_norm.weight.max":4.058460234546146e-10,"optim/param/transformer.blocks.5.attn_norm.weight.max":0.6824473738670349,"optim/grad/transformer.blocks.11.att_proj.weight.norm":0.00508095882833004,"optim/exp_avg/transformer.blocks.10.ffn.experts.mlp.w2.avg":1.7070424934706452e-11,"optim/grad/transformer.blocks.1.attn_norm.weight.min":7.845073923817836e-09,"optim/exp_avg_sq/transformer.blocks.13.k_norm.weight.avg":3.9439992070844365e-12,"optim/exp_avg_sq/transformer.blocks.9.attn_norm.weight.norm":9.982835891264585e-09,"optim/exp_avg/transformer.blocks.9.k_norm.weight.min":1.2164851070917138e-10,"optim/grad/transformer.blocks.6.attn_out.weight.norm":0.00520684290677309,"optim/grad/transformer.blocks.6.ffn.experts.mlp.w1.min":0,"optim/exp_avg_sq/transformer.blocks.9.k_norm.weight.avg":1.687049631937665e-11,"optim/grad/transformer.blocks.9.ffn.experts.mlp.w2.max":4.219645688863238e-06,"optim/grad/transformer.blocks.12.k_norm.weight.norm":9.811032214201987e-05,"optim/grad/transformer.blocks.12.ffn.experts.mlp.w2.avg":9.70031485691969e-11,"optim/exp_avg_sq/transformer.blocks.12.ffn.experts.mlp.w2.avg":2.6352845009262535e-14,"optim/exp_avg_sq/transformer.blocks.4.ff_norm.weight.norm":3.489395661304684e-09,"optim/exp_avg_sq/transformer.blocks.6.ffn.router.layer.weight.avg":6.10840822368175e-11,"optim/param/transformer.blocks.12.k_norm.weight.max":1.4153945446014404,"optim/exp_avg/transformer.blocks.5.attn_norm.weight.min":1.7152924913776957e-10,"optim/exp_avg_sq/transformer.blocks.2.attn_out.weight.norm":3.4252288116931595e-08,"optim/param/transformer.blocks.10.q_norm.weight.min":0.26122474670410156,"optim/param/transformer.ff_out.weight.min":2.5156179916940857e-10,"optim/param/transformer.blocks.7.k_norm.weight.norm":40.609230041503906,"optim/param/transformer.blocks.1.attn_norm.weight.max":0.6002467274665833,"optim/grad/transformer.wte.weight.avg":1.920780201558614e-10,"optim/grad/transformer.blocks.1.ffn.experts.mlp.w2.norm":0.0027860477566719055,"optim/grad/transformer.blocks.6.ffn.router.layer.weight.min":1.0231815394945443e-11,"optim/param/transformer.blocks.14.attn_out.weight.min":3.3618469119645056e-10,"optim/exp_avg/transformer.blocks.15.k_norm.weight.avg":-4.7127187485784816e-08,"optim/exp_avg/transformer.blocks.7.ffn.experts.mlp.w2.max":1.2145237633376382e-06,"optim/exp_avg/transformer.blocks.5.attn_out.weight.avg":7.239432386896283e-10,"optim/exp_avg/transformer.blocks.10.attn_out.weight.max":2.862863539121463e-06,"optim/param/transformer.blocks.1.ffn.experts.mlp.w2.min":2.764694029266934e-10,"optim/exp_avg/transformer.blocks.8.ffn.experts.mlp.w1.min":1.0656349985234534e-15,"optim/exp_avg/transformer.wte.weight.avg":5.858255547330771e-11,"eval/m2d2_s2orc-validation/CrossEntropyLoss":3.156592845916748,"optim/exp_avg/transformer.blocks.7.ffn.experts.mlp.w1.max":9.982587471313309e-06,"optim/grad/transformer.blocks.10.attn_norm.weight.avg":4.312234125336545e-09,"optim/param/transformer.blocks.3.k_norm.weight.avg":0.9517265558242798,"optim/grad/transformer.blocks.0.k_norm.weight.max":9.054284419107717e-06,"optim/param/transformer.blocks.11.ffn.experts.mlp.w1.max":0.25063878297805786,"optim/grad/transformer.blocks.4.k_norm.weight.avg":-1.2575522134739003e-07,"optim/grad/transformer.blocks.10.k_norm.weight.avg":-9.190015504145777e-08,"eval/downstream/mmlu_stem_mc_5shot_len_norm":0.2429906576871872,"optim/exp_avg/transformer.blocks.11.attn_out.weight.max":2.290770908075501e-06,"optim/exp_avg/transformer.blocks.2.attn_norm.weight.avg":1.6723389251183107e-07,"optim/grad/transformer.blocks.1.ffn.experts.mlp.w2.min":0,"optim/param/transformer.blocks.0.q_norm.weight.norm":36.7398567199707,"optim/param/transformer.blocks.8.ffn.experts.mlp.w2.max":0.2843891978263855,"optim/param/transformer.blocks.4.attn_norm.weight.norm":26.46015167236328,"optim/grad/transformer.blocks.13.k_norm.weight.avg":-1.848546737903689e-08,"optim/exp_avg_sq/transformer.blocks.11.k_norm.weight.norm":8.04738997661758e-10,"optim/exp_avg_sq/transformer.blocks.0.attn_out.weight.avg":8.5509273273221e-12,"optim/param/transformer.blocks.11.attn_out.weight.min":2.5948972748324195e-08,"optim/grad/transformer.blocks.12.att_proj.weight.norm":0.004338949453085661,"optim/grad/transformer.blocks.12.q_norm.weight.avg":-3.614468369050883e-07,"optim/grad/transformer.blocks.1.k_norm.weight.max":2.363307066843845e-05,"optim/exp_avg/transformer.blocks.14.k_norm.weight.avg":-7.795072320959662e-08,"optim/param/transformer.blocks.14.q_norm.weight.norm":42.177181243896484,"optim/exp_avg/transformer.blocks.8.att_proj.weight.avg":1.6788577472670596e-10,"optim/param/transformer.blocks.3.ffn.experts.mlp.w1.norm":318.48089599609375,"optim/exp_avg_sq/transformer.blocks.1.attn_norm.weight.avg":1.2317928621552454e-10,"optim/exp_avg/transformer.ln_f.weight.avg":-2.5341219043184537e-06,"optim/grad/transformer.blocks.10.q_norm.weight.norm":0.00012365513248369098,"optim/exp_avg/transformer.blocks.15.q_norm.weight.max":1.724401499814121e-06,"optim/exp_avg/transformer.blocks.11.ffn.experts.mlp.w1.max":1.7650368135946337e-06,"optim/param/transformer.blocks.12.attn_norm.weight.min":0.4474029541015625,"optim/exp_avg_sq/transformer.blocks.5.k_norm.weight.avg":6.381678518962985e-11,"optim/param/transformer.blocks.12.ff_norm.weight.avg":1.1319888830184937,"optim/exp_avg_sq/transformer.blocks.9.attn_out.weight.norm":7.131958579265074e-09,"optim/exp_avg_sq/transformer.blocks.13.q_norm.weight.max":2.437634538665634e-10,"optim/param/transformer.blocks.9.ffn.router.layer.weight.max":0.12403564900159836,"optim/exp_avg/transformer.blocks.12.att_proj.weight.norm":0.000906413362827152,"optim/exp_avg/transformer.blocks.10.ffn.experts.mlp.w2.min":3.9271745569247735e-16,"optim/grad/transformer.blocks.12.ffn.experts.mlp.w1.avg":4.207485054807947e-11,"optim/exp_avg_sq/transformer.blocks.6.k_norm.weight.min":7.234992058311861e-15,"optim/exp_avg/transformer.blocks.12.att_proj.weight.avg":-4.061949734901482e-11,"optim/exp_avg_sq/transformer.blocks.6.ffn.router.layer.weight.min":8.054930854262121e-12,"optim/exp_avg_sq/transformer.blocks.3.attn_out.weight.max":1.137191896205536e-09,"optim/param/transformer.blocks.11.k_norm.weight.avg":1.0180867910385132,"optim/exp_avg/transformer.ff_out.weight.norm":0.0036950386129319668,"eval/downstream/mmlu_stem_var_len_norm":0.2719181180000305,"optim/exp_avg/transformer.blocks.11.att_proj.weight.norm":0.0010370038216933608,"optim/grad/transformer.blocks.7.attn_norm.weight.max":0.00015428068581968546,"optim/exp_avg_sq/transformer.blocks.1.ffn.experts.mlp.w1.max":5.206171049820796e-10,"optim/param/transformer.blocks.15.attn_norm.weight.norm":28.992895126342773,"optim/param/transformer.blocks.9.ffn.experts.mlp.w1.max":0.16819466650485992,"optim/exp_avg_sq/transformer.blocks.15.q_norm.weight.min":1.5154714327997548e-15,"optim/param/transformer.blocks.1.att_proj.weight.avg":-4.188731054455275e-06,"optim/grad/transformer.blocks.11.ffn.router.layer.weight.avg":-5.472955422192172e-11,"optim/grad/transformer.blocks.11.ff_norm.weight.max":1.390112993249204e-05,"optim/grad/transformer.blocks.3.att_proj.weight.norm":0.01658782549202442,"optim/grad/transformer.blocks.0.ffn.experts.mlp.w2.norm":0.001590370899066329,"optim/exp_avg_sq/transformer.blocks.1.att_proj.weight.avg":3.034662451883996e-11,"optim/exp_avg_sq/transformer.blocks.1.ffn.experts.mlp.w2.min":1.8914415118777977e-16,"optim/param/transformer.blocks.2.k_norm.weight.max":1.3583638668060303,"optim/exp_avg/transformer.blocks.11.q_norm.weight.avg":-1.1886088202572864e-07,"optim/exp_avg_sq/transformer.blocks.3.k_norm.weight.norm":1.991544262835987e-09,"optim/param/transformer.blocks.12.ffn.experts.mlp.w2.avg":-6.146982741483953e-06,"eval/c4_en-validation/Perplexity":40.372764587402344,"optim/exp_avg_sq/transformer.blocks.2.ffn.experts.mlp.w2.min":3.642121394514981e-15,"optim/grad/transformer.blocks.4.ffn.experts.mlp.w2.norm":0.0029045725241303444,"optim/param/transformer.ln_f.weight.avg":1.2390022277832031,"optim/grad/transformer.blocks.0.attn_out.weight.min":0,"optim/exp_avg/transformer.blocks.3.attn_out.weight.norm":0.0011901740217581391,"optim/exp_avg_sq/transformer.blocks.2.ff_norm.weight.max":3.7025871257867493e-09,"optim/grad/transformer.blocks.6.ffn.experts.mlp.w1.norm":0.002946691121906042,"optim/grad/transformer.blocks.8.attn_out.weight.norm":0.004567696712911129,"optim/grad/transformer.blocks.2.ffn.experts.mlp.w2.max":1.2664173482335173e-05,"eval/dolma_reddit-validation/CrossEntropyLoss":3.8659560680389404,"optim/exp_avg_sq/transformer.blocks.15.ffn.experts.mlp.w2.min":2.088914273338133e-17,"optim/param/transformer.blocks.11.ffn.experts.mlp.w2.min":3.6976931933052626e-12,"optim/exp_avg/transformer.blocks.10.att_proj.weight.norm":0.0012578866444528103,"optim/param/transformer.blocks.1.ffn.router.layer.weight.min":1.3615438554381853e-08,"optim/param/transformer.blocks.3.q_norm.weight.avg":0.9554349780082703,"optim/exp_avg/transformer.blocks.12.ffn.experts.mlp.w1.min":1.4674839193735853e-16,"optim/exp_avg_sq/transformer.blocks.2.ffn.experts.mlp.w1.max":4.3884409994809914e-10,"optim/param/transformer.blocks.13.att_proj.weight.norm":80.84622955322266,"optim/exp_avg/transformer.blocks.7.q_norm.weight.min":9.704206188621e-12,"optim/exp_avg/transformer.blocks.15.attn_out.weight.max":5.10845848111785e-06,"optim/exp_avg_sq/transformer.blocks.3.attn_norm.weight.max":5.689115312179638e-08,"optim/exp_avg/transformer.wte.weight.min":0,"optim/grad/transformer.blocks.11.attn_out.weight.max":1.0739644494606182e-05,"optim/param/transformer.blocks.1.ffn.experts.mlp.w1.avg":-5.008597327105235e-06,"optim/exp_avg/transformer.blocks.8.attn_out.weight.max":4.801563591172453e-06,"optim/param/transformer.blocks.0.att_proj.weight.norm":74.00260925292969,"optim/exp_avg_sq/transformer.blocks.2.q_norm.weight.min":1.3433447876212007e-14,"optim/grad/transformer.blocks.13.ff_norm.weight.max":1.3404640412773006e-05,"optim/exp_avg_sq/transformer.blocks.3.q_norm.weight.max":1.6303597361044808e-09,"eval/wikitext_103-validation/Perplexity":33.73135757446289,"optim/param/transformer.blocks.12.ffn.router.layer.weight.norm":7.791259765625,"optim/grad/transformer.blocks.5.ffn.router.layer.weight.norm":0.002984518650919199,"optim/grad/transformer.blocks.12.k_norm.weight.max":2.948847395600751e-05,"optim/exp_avg/transformer.blocks.13.k_norm.weight.avg":-8.710996723948483e-08,"optim/grad/transformer.blocks.6.ffn.experts.mlp.w2.min":0,"optim/param/transformer.blocks.8.ffn.experts.mlp.w2.avg":-1.595188041392248e-05,"optim/param/transformer.blocks.4.q_norm.weight.min":0.3749074339866638,"optim/exp_avg_sq/transformer.blocks.11.q_norm.weight.max":2.9131980161523074e-10,"optim/grad/transformer.blocks.1.ffn.experts.mlp.w1.min":0,"optim/grad/transformer.blocks.12.ffn.experts.mlp.w2.max":2.9913846901763463e-06,"optim/exp_avg_sq/transformer.blocks.14.q_norm.weight.max":1.4058094965907486e-10,"optim/param/transformer.blocks.4.attn_out.weight.min":2.4911863683030333e-09,"optim/grad/transformer.blocks.14.attn_norm.weight.max":4.158153024036437e-05,"optim/exp_avg_sq/transformer.blocks.12.k_norm.weight.avg":5.637680860343108e-12,"optim/exp_avg/transformer.blocks.1.att_proj.weight.max":0.0002690903493203223,"optim/grad/transformer.blocks.7.k_norm.weight.max":3.0519684514729306e-05,"optim/grad/transformer.blocks.10.q_norm.weight.avg":-9.255251143258647e-08,"optim/exp_avg/transformer.blocks.12.att_proj.weight.max":2.474207394698169e-05,"optim/grad/transformer.blocks.13.att_proj.weight.min":0,"optim/grad/transformer.blocks.2.att_proj.weight.max":0.0009571488481014967,"optim/param/transformer.wte.weight.max":0.16886095702648163,"optim/grad/transformer.blocks.4.ffn.experts.mlp.w2.max":8.670799616083968e-06,"optim/grad/transformer.blocks.6.ffn.router.layer.weight.max":0.00020856480114161968,"optim/exp_avg/transformer.blocks.1.q_norm.weight.norm":1.9777511624852195e-05,"optim/exp_avg_sq/transformer.blocks.4.ffn.experts.mlp.w1.min":4.054374818102932e-16,"optim/param/transformer.blocks.1.q_norm.weight.avg":0.8890053629875183,"optim/param/transformer.blocks.8.ff_norm.weight.max":1.0976232290267944,"optim/exp_avg/transformer.blocks.6.ff_norm.weight.avg":-2.1811410988448188e-07,"optim/exp_avg_sq/transformer.blocks.11.attn_out.weight.max":2.1424277282799942e-11,"optim/exp_avg/transformer.blocks.14.q_norm.weight.avg":-7.798059442620797e-08,"optim/param/transformer.blocks.11.attn_norm.weight.min":0.5172936916351318,"optim/exp_avg_sq/transformer.blocks.15.ffn.experts.mlp.w2.max":8.339819286551364e-12,"optim/param/transformer.blocks.13.attn_out.weight.norm":46.99838638305664,"optim/exp_avg_sq/transformer.blocks.5.attn_norm.weight.norm":7.2285857299903e-08,"optim/param/transformer.blocks.1.ffn.experts.mlp.w2.max":0.3546544313430786,"optim/exp_avg/transformer.blocks.12.attn_out.weight.min":6.777078000213188e-15,"optim/exp_avg/transformer.blocks.2.ffn.experts.mlp.w2.norm":0.0006753358757123351,"optim/param/transformer.blocks.3.ff_norm.weight.min":0.582955539226532,"eval/dolma_pes2o-validation/Perplexity":11.858994483947754,"optim/param/transformer.blocks.12.q_norm.weight.min":0.4102862477302551,"optim/exp_avg/transformer.blocks.4.q_norm.weight.min":1.5584562806925106e-10,"optim/param/transformer.blocks.0.ffn.experts.mlp.w2.norm":278.1943359375,"optim/exp_avg_sq/transformer.blocks.10.ffn.experts.mlp.w1.min":6.6061216581062075e-16,"optim/param/transformer.blocks.8.q_norm.weight.max":1.279106616973877,"eval/ice-validation/CrossEntropyLoss":3.193260669708252,"optim/param/transformer.blocks.8.ffn.router.layer.weight.min":4.069102885750908e-07,"optim/exp_avg_sq/transformer.blocks.5.ffn.experts.mlp.w1.min":7.549041911363247e-16,"optim/exp_avg_sq/transformer.blocks.15.ff_norm.weight.avg":1.3897816925068085e-11,"optim/exp_avg_sq/transformer.blocks.2.k_norm.weight.max":4.0251463273577315e-10,"optim/param/transformer.blocks.4.k_norm.weight.avg":0.9925576448440552,"optim/exp_avg/transformer.blocks.3.q_norm.weight.avg":-8.86519870846314e-08,"eval/dolma_stack-validation/CrossEntropyLoss":1.3468520641326904,"optim/param/transformer.blocks.8.ffn.experts.mlp.w2.min":7.060915047096827e-11,"optim/exp_avg/transformer.blocks.4.ff_norm.weight.min":4.181747309539574e-11,"optim/exp_avg/transformer.blocks.10.att_proj.weight.max":2.6358882678323425e-05,"optim/exp_avg/transformer.blocks.5.att_proj.weight.norm":0.0024966062046587467,"optim/grad/transformer.blocks.5.attn_out.weight.norm":0.005484991706907749,"optim/param/transformer.blocks.8.attn_norm.weight.norm":29.7940616607666,"optim/exp_avg/transformer.blocks.14.ffn.experts.mlp.w1.norm":0.0005394297186285257,"optim/param/transformer.blocks.8.ffn.router.layer.weight.avg":0.00023565380251966417,"optim/grad/transformer.blocks.15.ff_norm.weight.min":1.0541043593548238e-09,"optim/exp_avg_sq/transformer.blocks.11.k_norm.weight.max":3.147409555204206e-10,"optim/grad/transformer.blocks.9.attn_norm.weight.max":7.77526292949915e-05,"optim/grad/transformer.blocks.13.ffn.experts.mlp.w1.avg":1.3409175679546248e-11,"optim/exp_avg_sq/transformer.blocks.8.ffn.router.layer.weight.min":8.737693728277929e-12,"optim/exp_avg/transformer.blocks.11.att_proj.weight.max":2.2176162019604817e-05,"optim/exp_avg/transformer.blocks.1.attn_norm.weight.max":4.468064071261324e-05,"optim/exp_avg/transformer.blocks.3.ffn.router.layer.weight.avg":6.202887092854326e-10,"optim/exp_avg_sq/transformer.blocks.8.attn_norm.weight.avg":3.9890153680222085e-11,"optim/exp_avg/transformer.blocks.6.ffn.experts.mlp.w1.avg":-1.0505807161720337e-11,"optim/param/transformer.blocks.0.q_norm.weight.avg":0.808296799659729,"optim/param/transformer.blocks.10.attn_out.weight.norm":49.62427520751953,"optim/exp_avg/transformer.blocks.7.q_norm.weight.avg":-1.76775444060695e-07,"optim/grad/transformer.blocks.1.att_proj.weight.avg":3.6519809398782854e-10,"optim/param/transformer.blocks.15.q_norm.weight.min":0.625325620174408,"optim/param/transformer.blocks.10.ffn.experts.mlp.w2.norm":331.2845764160156,"optim/param/transformer.blocks.10.ffn.experts.mlp.w2.avg":-1.6225829313043505e-05,"optim/exp_avg/transformer.blocks.5.k_norm.weight.norm":3.8269165088422596e-05,"optim/param/transformer.blocks.12.ffn.router.layer.weight.max":0.1537744104862213,"optim/exp_avg_sq/transformer.blocks.8.k_norm.weight.avg":2.5156396063485964e-11,"optim/exp_avg/transformer.blocks.12.k_norm.weight.avg":-1.3885842520267033e-07,"optim/grad/transformer.blocks.5.k_norm.weight.min":2.8421709430404007e-12,"optim/param/transformer.blocks.5.ff_norm.weight.avg":0.9489870071411133,"optim/grad/transformer.blocks.11.attn_out.weight.min":0,"optim/param/transformer.blocks.8.q_norm.weight.min":0.3165152966976166,"optim/grad/transformer.blocks.3.attn_norm.weight.norm":0.0003244522958993912,"optim/exp_avg_sq/transformer.blocks.3.ffn.experts.mlp.w1.norm":6.9557759552196785e-09,"optim/exp_avg_sq/transformer.blocks.9.q_norm.weight.max":2.0337282968796444e-09,"optim/exp_avg/transformer.blocks.14.attn_out.weight.min":6.007024169535011e-14,"optim/exp_avg_sq/transformer.blocks.0.ffn.router.layer.weight.avg":2.395025150425667e-11,"optim/exp_avg/transformer.blocks.11.attn_out.weight.min":1.6909388182739273e-14,"optim/exp_avg/transformer.blocks.5.ffn.router.layer.weight.max":5.504400542122312e-05,"optim/param/transformer.blocks.13.q_norm.weight.norm":42.48368453979492,"eval/dolma_wiki-validation/Perplexity":32.44643783569336,"optim/exp_avg/transformer.blocks.2.ffn.experts.mlp.w1.avg":1.6843510960984354e-11,"optim/exp_avg/transformer.blocks.1.ffn.router.layer.weight.max":6.902201857883483e-05,"optim/param/transformer.blocks.0.attn_out.weight.avg":-6.287932592385914e-07,"optim/grad/transformer.blocks.0.att_proj.weight.max":0.0003299408417660743,"optim/grad/transformer.blocks.15.ff_norm.weight.norm":0.00021948011999484152,"optim/grad/transformer.blocks.12.k_norm.weight.min":6.248228601180017e-10,"optim/grad/transformer.blocks.14.ff_norm.weight.norm":0.00010015653242589906,"optim/exp_avg_sq/transformer.blocks.15.k_norm.weight.avg":3.3635583467778174e-12,"optim/grad/transformer.blocks.2.ffn.experts.mlp.w2.norm":0.003041275544092059,"optim/exp_avg/transformer.blocks.15.ff_norm.weight.min":8.321064948191292e-10,"optim/param/transformer.blocks.0.ffn.experts.mlp.w1.min":3.881236165792501e-10,"System/Peak GPU Memory (MB)":43855.97265625,"optim/grad/transformer.blocks.7.q_norm.weight.avg":-2.050246621365659e-07,"optim/exp_avg/transformer.blocks.10.attn_out.weight.min":1.1290392686253477e-14,"optim/exp_avg/transformer.blocks.13.q_norm.weight.min":5.4808754756940914e-11,"optim/exp_avg_sq/transformer.blocks.8.attn_out.weight.max":2.5097693368003604e-10,"optim/grad/transformer.blocks.5.ffn.experts.mlp.w2.norm":0.002753637032583356,"optim/param/transformer.blocks.4.q_norm.weight.max":1.4974064826965332,"optim/grad/transformer.blocks.6.q_norm.weight.avg":1.0731078248227277e-09,"optim/grad/transformer.blocks.0.attn_out.weight.norm":0.006161058321595192,"optim/exp_avg_sq/transformer.blocks.11.ffn.experts.mlp.w2.max":3.6867813489604995e-12,"optim/exp_avg/transformer.blocks.1.ff_norm.weight.min":2.0791103214068407e-12,"eval/downstream/winogrande_acc":0.49960535764694214,"optim/exp_avg/transformer.blocks.10.ffn.experts.mlp.w2.max":5.193391530156077e-07,"optim/exp_avg/transformer.blocks.14.att_proj.weight.max":1.5806219380465336e-05,"optim/grad/transformer.blocks.10.ffn.router.layer.weight.norm":0.002465427853167057,"optim/grad/transformer.blocks.5.att_proj.weight.norm":0.01780495047569275,"optim/exp_avg/transformer.blocks.9.ff_norm.weight.norm":2.0136434613959864e-05,"optim/grad/transformer.blocks.5.k_norm.weight.norm":0.0001406772353220731,"optim/param/transformer.blocks.2.ffn.experts.mlp.w1.avg":1.41740711114835e-05,"optim/exp_avg_sq/transformer.blocks.15.ffn.router.layer.weight.avg":3.7193290114423405e-11,"optim/param/transformer.blocks.12.ff_norm.weight.min":0.5003184080123901,"optim/grad/transformer.blocks.6.att_proj.weight.avg":-6.164412869047453e-10,"optim/exp_avg_sq/transformer.blocks.13.attn_norm.weight.avg":9.486793295376827e-12} \ No newline at end of file diff --git a/wandb/wandb/run-20251219_032809-fhag09gx/logs/debug-internal.log b/wandb/wandb/run-20251219_032809-fhag09gx/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..186f99d7e572e5cdbb6ee8916298db39d1ae396b --- /dev/null +++ b/wandb/wandb/run-20251219_032809-fhag09gx/logs/debug-internal.log @@ -0,0 +1,13 @@ +{"time":"2025-12-19T03:28:09.715068381Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"} +{"time":"2025-12-19T03:28:10.010671632Z","level":"INFO","msg":"stream: created new stream","id":"fhag09gx"} +{"time":"2025-12-19T03:28:10.01079177Z","level":"INFO","msg":"writer: started","stream_id":"fhag09gx"} +{"time":"2025-12-19T03:28:10.010792247Z","level":"INFO","msg":"sender: started","stream_id":"fhag09gx"} +{"time":"2025-12-19T03:28:10.011088687Z","level":"INFO","msg":"handler: started","stream_id":"fhag09gx"} +{"time":"2025-12-19T03:28:10.011242775Z","level":"INFO","msg":"stream: started","id":"fhag09gx"} +{"time":"2025-12-19T03:28:10.069884068Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"} +{"time":"2025-12-25T13:58:15.121256706Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-12-25T13:58:15.388114558Z","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2025-12-25T13:58:15.404292716Z","level":"INFO","msg":"stream: closing","id":"fhag09gx"} +{"time":"2025-12-25T13:58:15.404311317Z","level":"INFO","msg":"handler: closed","stream_id":"fhag09gx"} +{"time":"2025-12-25T13:58:15.404690567Z","level":"INFO","msg":"sender: closed","stream_id":"fhag09gx"} +{"time":"2025-12-25T13:58:15.404702005Z","level":"INFO","msg":"stream: closed","id":"fhag09gx"} diff --git a/wandb/wandb/run-20251219_032809-fhag09gx/logs/debug.log b/wandb/wandb/run-20251219_032809-fhag09gx/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..66e1e967bc1e3af2dfc513dfcb07874b1410f45f --- /dev/null +++ b/wandb/wandb/run-20251219_032809-fhag09gx/logs/debug.log @@ -0,0 +1,26 @@ +2025-12-19 03:28:09,456 INFO MainThread:1675715 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1 +2025-12-19 03:28:09,456 INFO MainThread:1675715 [wandb_setup.py:_flush():80] Configure stats pid to 1675715 +2025-12-19 03:28:09,456 INFO MainThread:1675715 [wandb_setup.py:_flush():80] Loading settings from /data/users/xandi281/.config/wandb/settings +2025-12-19 03:28:09,456 INFO MainThread:1675715 [wandb_setup.py:_flush():80] Loading settings from /data/users/xandi281/work/Hi-OLMoE/OLMo/wandb/settings +2025-12-19 03:28:09,456 INFO MainThread:1675715 [wandb_setup.py:_flush():80] Loading settings from environment variables +2025-12-19 03:28:09,456 INFO MainThread:1675715 [wandb_init.py:setup_run_log_directory():714] Logging user logs to runs/olmoe-7b-29b-2ep/wandb/wandb/run-20251219_032809-fhag09gx/logs/debug.log +2025-12-19 03:28:09,456 INFO MainThread:1675715 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to runs/olmoe-7b-29b-2ep/wandb/wandb/run-20251219_032809-fhag09gx/logs/debug-internal.log +2025-12-19 03:28:09,456 INFO MainThread:1675715 [wandb_init.py:init():841] calling init triggers +2025-12-19 03:28:09,457 INFO MainThread:1675715 [wandb_init.py:init():846] wandb.init called with sweep_config: {} +config: {'run_name': 'olmoe-7b-29b-2ep', 'seed': 6198, 'epoch': None, 'dry_run': False, 'model': {'d_model': 2048, 'n_heads': 16, 'n_kv_heads': None, 'clip_qkv': None, 'n_layers': 16, 'mlp_ratio': 1, 'mlp_hidden_size': None, 'activation_type': 'swiglu', 'block_type': 'moe', 'block_group_size': 1, 'alibi': False, 'alibi_bias_max': 8.0, 'rope': True, 'rope_full_precision': True, 'rope_theta': 10000, 'flash_attention': True, 'attention_dropout': 0.0, 'multi_query_attention': None, 'attention_layer_norm': True, 'residual_dropout': 0.0, 'embedding_dropout': 0.0, 'embedding_layer_norm': False, 'layer_norm_type': 'rms', 'layer_norm_with_affine': True, 'layer_norm_eps': 1e-05, 'attention_layer_norm_with_affine': True, 'max_sequence_length': 4096, 'include_bias': False, 'bias_for_layer_norm': False, 'scale_logits': False, 'vocab_size': 50280, 'embedding_size': 50304, 'weight_tying': False, 'eos_token_id': 0, 'pad_token_id': 1, 'init_device': 'meta', 'init_fn': 'normal', 'init_std': 0.02, 'init_cutoff_factor': 3.0, 'precision': 'amp_bf16', 'moe_num_experts': 64, 'moe_top_k': 8, 'moe_mlp_impl': 'sparse', 'moe_log_expert_assignment': False, 'moe_shared_expert': False, 'moe_lbl_in_fp32': False, 'moe_loss_weight': 0.01, 'moe_zloss_weight': 0.001, 'moe_dropless': False, 'moe_capacity_factor': 1.25, 'himoe_enabled': False, 'himoe_n_groups': 4, 'himoe_top_k_per_group': 2, 'himoe_lambda1': 0.01, 'himoe_lambda2': 0.1, 'scale_emb_init': False, 'emb_init_std': None, 'norm_after': False}, 'optimizer': {'name': 'adamw', 'learning_rate': 0.0004, 'weight_decay': 0.1, 'betas': (0.9, 0.95), 'eps': 1e-08, 'no_decay_norm_and_bias': None, 'selective_updates': False, 'decay_norm_and_bias': True, 'decay_embeddings': True, 'metrics_log_interval': 10, 'record_update_metrics': False}, 'scheduler': {'name': 'cosine_with_warmup', 'units': 'tokens', 't_warmup': 1405296000, 't_max': 58540000000, 'alpha_f': 0.1, 'grad_clip_warmup_steps': None, 'grad_clip_warmup_factor': None, 'warmup_min_lr': None}, 'data': {'paths': ['/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-13-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-14-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/algebraic-stack/allenai/gpt-neox-olmo-dolma-v1_5/part-15-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-000-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-001-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-002-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-003-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-004-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-005-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-006-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-007-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-008-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-009-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-010-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-011-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-012-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/arxiv/allenai/gpt-neox-olmo-dolma-v1_5/part-013-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy', '/data/users/xandi281/work/Hi-OLMoE/data/training/open-web-math/allenai/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy'], 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 8, 'drop_last': True, 'pin_memory': True, 'prefetch_factor': 8, 'persistent_workers': True, 'timeout': 0, 'seed': None, 'instance_filter': {'repetition_max_period': 13, 'repetition_min_period': 1, 'repetition_max_count': 32}}, 'restore_dataloader': True, 'fast_forward_batches': None, 'evaluators': [{'label': 'all-small-ppl-validation', 'type': 'lm', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': {'c4_en-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/c4_en/val/part-0-00000.npy'], 'dolma_books-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_books/val/part-0-00000.npy'], 'dolma_common-crawl-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_common-crawl/val/part-0-00000.npy'], 'dolma_pes2o-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_pes2o/val/part-0-00000.npy'], 'dolma_reddit-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_reddit/val/part-0-00000.npy'], 'dolma_stack-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_stack/val/part-0-00000.npy'], 'dolma_wiki-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/dolma_wiki/val/part-0-00000.npy'], 'ice-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/ice/val/part-0-00000.npy'], 'm2d2_s2orc-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/m2d2_s2orc/val/part-0-00000.npy'], 'pile-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/pile/val/part-0-00000.npy'], 'wikitext_103-validation': ['/data/users/xandi281/work/Hi-OLMoE/data/eval-data/perplexity/v3_small_gptneox20b/wikitext_103/val/part-0-00000.npy']}, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': True, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'piqa', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'hellaswag', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'winogrande', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'openbook_qa', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'boolq', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'sciq', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'arc_easy', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'arc_challenge', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'copa', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'commonsense_qa', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'social_iqa', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_stem_var', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_humanities_var', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_social_sciences_var', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_other_var', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_stem_mc_5shot', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_humanities_mc_5shot', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_social_sciences_mc_5shot', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_other_mc_5shot', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_stem_mc_5shot_test', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_humanities_mc_5shot_test', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_social_sciences_mc_5shot_test', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}, {'label': 'mmlu_other_mc_5shot_test', 'type': 'downstream', 'data': {'paths': None, 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 0, 'drop_last': False, 'pin_memory': False, 'prefetch_factor': None, 'persistent_workers': False, 'timeout': 0, 'seed': None, 'instance_filter': None}, 'device_eval_batch_size': None, 'subset_num_batches': None}], 'eval_interval': 2000, 'tokenizer': {'identifier': 'tokenizers/allenai_gpt-neox-olmo-dolma-v1_5.json', 'truncate_direction': 'right'}, 'save_folder': 'runs/olmoe-7b-29b-2ep', 'remote_save_folder': None, 'canceled_check_interval': 50, 'save_interval': 2000, 'save_interval_unsharded': None, 'save_interval_ephemeral': None, 'save_num_checkpoints_to_keep': 5, 'save_num_unsharded_checkpoints_to_keep': -1, 'save_overwrite': True, 'force_save_unsharded': False, 'no_pre_train_checkpoint': True, 'load_path': None, 'load_path_sharded_checkpointer': None, 'try_load_latest_save': False, 'reset_optimizer_state': False, 'reset_trainer_state': False, 'sharded_checkpointer': 'olmo_core', 'new_style_checkpoints': None, 'max_duration': '2ep', 'global_train_batch_size': 1024, 'device_train_batch_size': 256, 'device_train_microbatch_size': 2, 'device_eval_batch_size': 1, 'eval_subset_num_batches': -1, 'eval_on_load': False, 'device_train_grad_accum': 128, 'max_grad_norm': 1.0, 'max_grad_norm_ratio': None, 'precision': 'amp_bf16', 'speed_monitor': {'window_size': 1, 'gpu_flops_available': None}, 'console_log_interval': 1, 'gen1_gc_interval': 1, 'compile': None, 'distributed_strategy': 'fsdp', 'fsdp': {'use_orig_params': True, 'sharding_strategy': , 'wrapping_strategy': 'by_block', 'precision': 'mixed', 'hybrid_sharding_num_model_replicas': None}, 'ddp': None, 'softmax_auxiliary_loss': False, 'auxiliary_loss_multiplier': 0.0001, 'time_limit': None, 'extra_steps_after_cancel': 10, 'early_stopping_factor': None, 'save_data_indices': True, 'python_profiling': False, 'torch_profiling': False, 'stop_at': None, 'stop_after': None, 'activation_checkpointing': 'fine_grained', 'fused_loss': None, 'hf_datasets_cache_dir': None, 'module_outputs_save_steps': None, '_wandb': {'code_path': 'code/OLMo/scripts/train.py'}} +2025-12-19 03:28:09,457 INFO MainThread:1675715 [wandb_init.py:init():889] starting backend +2025-12-19 03:28:09,710 INFO MainThread:1675715 [wandb_init.py:init():892] sending inform_init request +2025-12-19 03:28:09,713 INFO MainThread:1675715 [wandb_init.py:init():900] backend started and connected +2025-12-19 03:28:09,716 INFO MainThread:1675715 [wandb_init.py:init():970] updated telemetry +2025-12-19 03:28:09,718 INFO MainThread:1675715 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout +2025-12-19 03:28:10,068 INFO MainThread:1675715 [wandb_init.py:init():1041] starting run threads in backend +2025-12-19 03:28:10,155 INFO MainThread:1675715 [wandb_run.py:_console_start():2521] atexit reg +2025-12-19 03:28:10,155 INFO MainThread:1675715 [wandb_run.py:_redirect():2369] redirect: wrap_raw +2025-12-19 03:28:10,155 INFO MainThread:1675715 [wandb_run.py:_redirect():2438] Wrapping output streams. +2025-12-19 03:28:10,155 INFO MainThread:1675715 [wandb_run.py:_redirect():2461] Redirects installed. +2025-12-19 03:28:10,158 INFO MainThread:1675715 [wandb_init.py:init():1081] run started, returning control to user process +2025-12-25 13:58:11,885 INFO MainThread:1675715 [wandb_run.py:_finish():2287] finishing run bulgakov/hiomoe/fhag09gx +2025-12-25 13:58:11,885 INFO MainThread:1675715 [wandb_run.py:_atexit_cleanup():2486] got exitcode: 0 +2025-12-25 13:58:11,885 INFO MainThread:1675715 [wandb_run.py:_restore():2468] restore +2025-12-25 13:58:11,885 INFO MainThread:1675715 [wandb_run.py:_restore():2474] restore done +2025-12-25 13:58:15,403 INFO MainThread:1675715 [wandb_run.py:_footer_sync_info():3862] logging synced files