diff --git "a/summary.md" "b/summary.md" --- "a/summary.md" +++ "b/summary.md" @@ -31,7 +31,7 @@ Statistic mode: `signed` Sublayer granularity: `module` -Include non-layer parameters: `True` +Include non-layer parameters: `False` Device: `cuda:0` @@ -51,511 +51,509 @@ Summary top-k per model: `50` ## Compared parameter tensors -- `xplain1.3-6base-16k-step2199`: 362 parameter tensors -- `xplain1.3-6base-32k-step1647`: 362 parameter tensors -- `xplain1.3-6base-64k-step1647`: 362 parameter tensors -- `xplain1.3-6base-8k-step2199`: 362 parameter tensors -- `xplain1.4-6base-16k-step2319`: 362 parameter tensors -- `xplain1.4-6base-32k-step1743`: 362 parameter tensors -- `xplain1.4-6base-64k-step1743`: 362 parameter tensors -- `xplain1.4-6base-8k-step2319`: 362 parameter tensors -- `xplain1.4-7base-32k-step1743`: 362 parameter tensors +- `xplain1.3-6base-16k-step2199`: 360 parameter tensors +- `xplain1.3-6base-32k-step1647`: 360 parameter tensors +- `xplain1.3-6base-64k-step1647`: 360 parameter tensors +- `xplain1.3-6base-8k-step2199`: 360 parameter tensors +- `xplain1.4-6base-16k-step2319`: 360 parameter tensors +- `xplain1.4-6base-32k-step1743`: 360 parameter tensors +- `xplain1.4-6base-64k-step1743`: 360 parameter tensors +- `xplain1.4-6base-8k-step2319`: 360 parameter tensors +- `xplain1.4-7base-32k-step1743`: 360 parameter tensors ## Top sublayers by std ### `xplain1.3-6base-16k-step2199` - model sublayer block_label module_type count std stat_mode -xplain1.3-6base-16k-step2199 Layer 22 | input_layernorm Layer 22 input_layernorm 4096 0.003985 signed -xplain1.3-6base-16k-step2199 Final Norm | model.norm.weight Final Norm final_norm 4096 0.003860 signed -xplain1.3-6base-16k-step2199 Layer 25 | input_layernorm Layer 25 input_layernorm 4096 0.003845 signed -xplain1.3-6base-16k-step2199 Layer 28 | input_layernorm Layer 28 input_layernorm 4096 0.003739 signed -xplain1.3-6base-16k-step2199 Layer 18 | input_layernorm Layer 18 input_layernorm 4096 0.002777 signed -xplain1.3-6base-16k-step2199 Layer 21 | input_layernorm Layer 21 input_layernorm 4096 0.002399 signed -xplain1.3-6base-16k-step2199 Layer 38 | input_layernorm Layer 38 input_layernorm 4096 0.001991 signed -xplain1.3-6base-16k-step2199 Layer 37 | mlp.gate_proj Layer 37 mlp.gate_proj 52428800 0.001628 signed -xplain1.3-6base-16k-step2199 Layer 38 | mlp.gate_proj Layer 38 mlp.gate_proj 52428800 0.001616 signed -xplain1.3-6base-16k-step2199 Layer 36 | mlp.gate_proj Layer 36 mlp.gate_proj 52428800 0.001615 signed -xplain1.3-6base-16k-step2199 Layer 37 | mlp.up_proj Layer 37 mlp.up_proj 52428800 0.001575 signed -xplain1.3-6base-16k-step2199 Layer 36 | mlp.up_proj Layer 36 mlp.up_proj 52428800 0.001559 signed -xplain1.3-6base-16k-step2199 Layer 17 | input_layernorm Layer 17 input_layernorm 4096 0.001556 signed -xplain1.3-6base-16k-step2199 Layer 34 | mlp.gate_proj Layer 34 mlp.gate_proj 52428800 0.001552 signed -xplain1.3-6base-16k-step2199 Layer 38 | mlp.up_proj Layer 38 mlp.up_proj 52428800 0.001551 signed -xplain1.3-6base-16k-step2199 Layer 35 | mlp.gate_proj Layer 35 mlp.gate_proj 52428800 0.001546 signed -xplain1.3-6base-16k-step2199 Embedding | model.embed_tokens.weight Embedding embed_tokens 201326592 0.001496 signed -xplain1.3-6base-16k-step2199 Layer 34 | mlp.up_proj Layer 34 mlp.up_proj 52428800 0.001495 signed -xplain1.3-6base-16k-step2199 Layer 00 | input_layernorm Layer 00 input_layernorm 4096 0.001493 signed -xplain1.3-6base-16k-step2199 Layer 36 | input_layernorm Layer 36 input_layernorm 4096 0.001488 signed -xplain1.3-6base-16k-step2199 Layer 35 | mlp.up_proj Layer 35 mlp.up_proj 52428800 0.001483 signed -xplain1.3-6base-16k-step2199 Layer 39 | mlp.gate_proj Layer 39 mlp.gate_proj 52428800 0.001481 signed -xplain1.3-6base-16k-step2199 Layer 36 | self_attn.q_proj Layer 36 self_attn.q_proj 16777216 0.001466 signed -xplain1.3-6base-16k-step2199 Layer 35 | input_layernorm Layer 35 input_layernorm 4096 0.001441 signed -xplain1.3-6base-16k-step2199 Layer 38 | self_attn.q_proj Layer 38 self_attn.q_proj 16777216 0.001440 signed -xplain1.3-6base-16k-step2199 Layer 39 | self_attn.q_proj Layer 39 self_attn.q_proj 16777216 0.001434 signed -xplain1.3-6base-16k-step2199 Layer 35 | self_attn.q_proj Layer 35 self_attn.q_proj 16777216 0.001433 signed -xplain1.3-6base-16k-step2199 Layer 34 | mlp.down_proj Layer 34 mlp.down_proj 52428800 0.001420 signed -xplain1.3-6base-16k-step2199 Layer 37 | input_layernorm Layer 37 input_layernorm 4096 0.001418 signed -xplain1.3-6base-16k-step2199 Layer 36 | mlp.down_proj Layer 36 mlp.down_proj 52428800 0.001414 signed -xplain1.3-6base-16k-step2199 Layer 33 | mlp.gate_proj Layer 33 mlp.gate_proj 52428800 0.001409 signed -xplain1.3-6base-16k-step2199 Layer 37 | self_attn.q_proj Layer 37 self_attn.q_proj 16777216 0.001401 signed -xplain1.3-6base-16k-step2199 Layer 34 | self_attn.q_proj Layer 34 self_attn.q_proj 16777216 0.001398 signed -xplain1.3-6base-16k-step2199 Layer 37 | mlp.down_proj Layer 37 mlp.down_proj 52428800 0.001396 signed -xplain1.3-6base-16k-step2199 Layer 31 | mlp.gate_proj Layer 31 mlp.gate_proj 52428800 0.001388 signed -xplain1.3-6base-16k-step2199 Layer 32 | mlp.gate_proj Layer 32 mlp.gate_proj 52428800 0.001375 signed -xplain1.3-6base-16k-step2199 Layer 35 | mlp.down_proj Layer 35 mlp.down_proj 52428800 0.001369 signed -xplain1.3-6base-16k-step2199 Layer 38 | mlp.down_proj Layer 38 mlp.down_proj 52428800 0.001368 signed -xplain1.3-6base-16k-step2199 Layer 39 | mlp.up_proj Layer 39 mlp.up_proj 52428800 0.001366 signed -xplain1.3-6base-16k-step2199 Layer 30 | mlp.gate_proj Layer 30 mlp.gate_proj 52428800 0.001363 signed -xplain1.3-6base-16k-step2199 Layer 29 | mlp.gate_proj Layer 29 mlp.gate_proj 52428800 0.001358 signed -xplain1.3-6base-16k-step2199 Layer 28 | mlp.gate_proj Layer 28 mlp.gate_proj 52428800 0.001356 signed -xplain1.3-6base-16k-step2199 Layer 21 | self_attn.q_proj Layer 21 self_attn.q_proj 16777216 0.001356 signed -xplain1.3-6base-16k-step2199 Layer 27 | mlp.gate_proj Layer 27 mlp.gate_proj 52428800 0.001351 signed -xplain1.3-6base-16k-step2199 Layer 36 | self_attn.k_proj Layer 36 self_attn.k_proj 4194304 0.001351 signed -xplain1.3-6base-16k-step2199 Layer 33 | mlp.up_proj Layer 33 mlp.up_proj 52428800 0.001349 signed -xplain1.3-6base-16k-step2199 Layer 29 | mlp.down_proj Layer 29 mlp.down_proj 52428800 0.001347 signed -xplain1.3-6base-16k-step2199 Layer 31 | mlp.down_proj Layer 31 mlp.down_proj 52428800 0.001344 signed -xplain1.3-6base-16k-step2199 Layer 31 | mlp.up_proj Layer 31 mlp.up_proj 52428800 0.001343 signed -xplain1.3-6base-16k-step2199 Layer 26 | mlp.gate_proj Layer 26 mlp.gate_proj 52428800 0.001343 signed + model sublayer block_label module_type count std stat_mode +xplain1.3-6base-16k-step2199 Layer 22 | input_layernorm Layer 22 input_layernorm 4096 0.003985 signed +xplain1.3-6base-16k-step2199 Layer 25 | input_layernorm Layer 25 input_layernorm 4096 0.003845 signed +xplain1.3-6base-16k-step2199 Layer 28 | input_layernorm Layer 28 input_layernorm 4096 0.003739 signed +xplain1.3-6base-16k-step2199 Layer 18 | input_layernorm Layer 18 input_layernorm 4096 0.002777 signed +xplain1.3-6base-16k-step2199 Layer 21 | input_layernorm Layer 21 input_layernorm 4096 0.002399 signed +xplain1.3-6base-16k-step2199 Layer 38 | input_layernorm Layer 38 input_layernorm 4096 0.001991 signed +xplain1.3-6base-16k-step2199 Layer 37 | mlp.gate_proj Layer 37 mlp.gate_proj 52428800 0.001628 signed +xplain1.3-6base-16k-step2199 Layer 38 | mlp.gate_proj Layer 38 mlp.gate_proj 52428800 0.001616 signed +xplain1.3-6base-16k-step2199 Layer 36 | mlp.gate_proj Layer 36 mlp.gate_proj 52428800 0.001615 signed +xplain1.3-6base-16k-step2199 Layer 37 | mlp.up_proj Layer 37 mlp.up_proj 52428800 0.001575 signed +xplain1.3-6base-16k-step2199 Layer 36 | mlp.up_proj Layer 36 mlp.up_proj 52428800 0.001559 signed +xplain1.3-6base-16k-step2199 Layer 17 | input_layernorm Layer 17 input_layernorm 4096 0.001556 signed +xplain1.3-6base-16k-step2199 Layer 34 | mlp.gate_proj Layer 34 mlp.gate_proj 52428800 0.001552 signed +xplain1.3-6base-16k-step2199 Layer 38 | mlp.up_proj Layer 38 mlp.up_proj 52428800 0.001551 signed +xplain1.3-6base-16k-step2199 Layer 35 | mlp.gate_proj Layer 35 mlp.gate_proj 52428800 0.001546 signed +xplain1.3-6base-16k-step2199 Layer 34 | mlp.up_proj Layer 34 mlp.up_proj 52428800 0.001495 signed +xplain1.3-6base-16k-step2199 Layer 00 | input_layernorm Layer 00 input_layernorm 4096 0.001493 signed +xplain1.3-6base-16k-step2199 Layer 36 | input_layernorm Layer 36 input_layernorm 4096 0.001488 signed +xplain1.3-6base-16k-step2199 Layer 35 | mlp.up_proj Layer 35 mlp.up_proj 52428800 0.001483 signed +xplain1.3-6base-16k-step2199 Layer 39 | mlp.gate_proj Layer 39 mlp.gate_proj 52428800 0.001481 signed +xplain1.3-6base-16k-step2199 Layer 36 | self_attn.q_proj Layer 36 self_attn.q_proj 16777216 0.001466 signed +xplain1.3-6base-16k-step2199 Layer 35 | input_layernorm Layer 35 input_layernorm 4096 0.001441 signed +xplain1.3-6base-16k-step2199 Layer 38 | self_attn.q_proj Layer 38 self_attn.q_proj 16777216 0.001440 signed +xplain1.3-6base-16k-step2199 Layer 39 | self_attn.q_proj Layer 39 self_attn.q_proj 16777216 0.001434 signed +xplain1.3-6base-16k-step2199 Layer 35 | self_attn.q_proj Layer 35 self_attn.q_proj 16777216 0.001433 signed +xplain1.3-6base-16k-step2199 Layer 34 | mlp.down_proj Layer 34 mlp.down_proj 52428800 0.001420 signed +xplain1.3-6base-16k-step2199 Layer 37 | input_layernorm Layer 37 input_layernorm 4096 0.001418 signed +xplain1.3-6base-16k-step2199 Layer 36 | mlp.down_proj Layer 36 mlp.down_proj 52428800 0.001414 signed +xplain1.3-6base-16k-step2199 Layer 33 | mlp.gate_proj Layer 33 mlp.gate_proj 52428800 0.001409 signed +xplain1.3-6base-16k-step2199 Layer 37 | self_attn.q_proj Layer 37 self_attn.q_proj 16777216 0.001401 signed +xplain1.3-6base-16k-step2199 Layer 34 | self_attn.q_proj Layer 34 self_attn.q_proj 16777216 0.001398 signed +xplain1.3-6base-16k-step2199 Layer 37 | mlp.down_proj Layer 37 mlp.down_proj 52428800 0.001396 signed +xplain1.3-6base-16k-step2199 Layer 31 | mlp.gate_proj Layer 31 mlp.gate_proj 52428800 0.001388 signed +xplain1.3-6base-16k-step2199 Layer 32 | mlp.gate_proj Layer 32 mlp.gate_proj 52428800 0.001375 signed +xplain1.3-6base-16k-step2199 Layer 35 | mlp.down_proj Layer 35 mlp.down_proj 52428800 0.001369 signed +xplain1.3-6base-16k-step2199 Layer 38 | mlp.down_proj Layer 38 mlp.down_proj 52428800 0.001368 signed +xplain1.3-6base-16k-step2199 Layer 39 | mlp.up_proj Layer 39 mlp.up_proj 52428800 0.001366 signed +xplain1.3-6base-16k-step2199 Layer 30 | mlp.gate_proj Layer 30 mlp.gate_proj 52428800 0.001363 signed +xplain1.3-6base-16k-step2199 Layer 29 | mlp.gate_proj Layer 29 mlp.gate_proj 52428800 0.001358 signed +xplain1.3-6base-16k-step2199 Layer 28 | mlp.gate_proj Layer 28 mlp.gate_proj 52428800 0.001356 signed +xplain1.3-6base-16k-step2199 Layer 21 | self_attn.q_proj Layer 21 self_attn.q_proj 16777216 0.001356 signed +xplain1.3-6base-16k-step2199 Layer 27 | mlp.gate_proj Layer 27 mlp.gate_proj 52428800 0.001351 signed +xplain1.3-6base-16k-step2199 Layer 36 | self_attn.k_proj Layer 36 self_attn.k_proj 4194304 0.001351 signed +xplain1.3-6base-16k-step2199 Layer 33 | mlp.up_proj Layer 33 mlp.up_proj 52428800 0.001349 signed +xplain1.3-6base-16k-step2199 Layer 29 | mlp.down_proj Layer 29 mlp.down_proj 52428800 0.001347 signed +xplain1.3-6base-16k-step2199 Layer 31 | mlp.down_proj Layer 31 mlp.down_proj 52428800 0.001344 signed +xplain1.3-6base-16k-step2199 Layer 31 | mlp.up_proj Layer 31 mlp.up_proj 52428800 0.001343 signed +xplain1.3-6base-16k-step2199 Layer 26 | mlp.gate_proj Layer 26 mlp.gate_proj 52428800 0.001343 signed +xplain1.3-6base-16k-step2199 Layer 28 | mlp.down_proj Layer 28 mlp.down_proj 52428800 0.001340 signed +xplain1.3-6base-16k-step2199 Layer 30 | mlp.down_proj Layer 30 mlp.down_proj 52428800 0.001340 signed ### `xplain1.3-6base-32k-step1647` - model sublayer block_label module_type count std stat_mode -xplain1.3-6base-32k-step1647 Layer 22 | input_layernorm Layer 22 input_layernorm 4096 0.005447 signed -xplain1.3-6base-32k-step1647 Final Norm | model.norm.weight Final Norm final_norm 4096 0.005347 signed -xplain1.3-6base-32k-step1647 Layer 25 | input_layernorm Layer 25 input_layernorm 4096 0.005222 signed -xplain1.3-6base-32k-step1647 Layer 28 | input_layernorm Layer 28 input_layernorm 4096 0.005059 signed -xplain1.3-6base-32k-step1647 Layer 18 | input_layernorm Layer 18 input_layernorm 4096 0.003657 signed -xplain1.3-6base-32k-step1647 Layer 21 | input_layernorm Layer 21 input_layernorm 4096 0.003223 signed -xplain1.3-6base-32k-step1647 Layer 38 | input_layernorm Layer 38 input_layernorm 4096 0.002641 signed -xplain1.3-6base-32k-step1647 Layer 36 | input_layernorm Layer 36 input_layernorm 4096 0.001945 signed -xplain1.3-6base-32k-step1647 Layer 17 | input_layernorm Layer 17 input_layernorm 4096 0.001918 signed -xplain1.3-6base-32k-step1647 Layer 00 | input_layernorm Layer 00 input_layernorm 4096 0.001915 signed -xplain1.3-6base-32k-step1647 Layer 37 | mlp.gate_proj Layer 37 mlp.gate_proj 52428800 0.001854 signed -xplain1.3-6base-32k-step1647 Layer 37 | input_layernorm Layer 37 input_layernorm 4096 0.001846 signed -xplain1.3-6base-32k-step1647 Layer 38 | mlp.gate_proj Layer 38 mlp.gate_proj 52428800 0.001841 signed -xplain1.3-6base-32k-step1647 Layer 36 | mlp.gate_proj Layer 36 mlp.gate_proj 52428800 0.001839 signed -xplain1.3-6base-32k-step1647 Layer 35 | input_layernorm Layer 35 input_layernorm 4096 0.001798 signed -xplain1.3-6base-32k-step1647 Layer 37 | mlp.up_proj Layer 37 mlp.up_proj 52428800 0.001796 signed -xplain1.3-6base-32k-step1647 Layer 36 | mlp.up_proj Layer 36 mlp.up_proj 52428800 0.001786 signed -xplain1.3-6base-32k-step1647 Layer 34 | mlp.gate_proj Layer 34 mlp.gate_proj 52428800 0.001780 signed -xplain1.3-6base-32k-step1647 Layer 35 | mlp.gate_proj Layer 35 mlp.gate_proj 52428800 0.001771 signed -xplain1.3-6base-32k-step1647 Layer 38 | mlp.up_proj Layer 38 mlp.up_proj 52428800 0.001767 signed -xplain1.3-6base-32k-step1647 Embedding | model.embed_tokens.weight Embedding embed_tokens 201326592 0.001762 signed -xplain1.3-6base-32k-step1647 Layer 34 | mlp.up_proj Layer 34 mlp.up_proj 52428800 0.001721 signed -xplain1.3-6base-32k-step1647 Layer 35 | mlp.up_proj Layer 35 mlp.up_proj 52428800 0.001705 signed -xplain1.3-6base-32k-step1647 Layer 39 | mlp.gate_proj Layer 39 mlp.gate_proj 52428800 0.001701 signed -xplain1.3-6base-32k-step1647 Layer 36 | self_attn.q_proj Layer 36 self_attn.q_proj 16777216 0.001669 signed -xplain1.3-6base-32k-step1647 Layer 30 | input_layernorm Layer 30 input_layernorm 4096 0.001663 signed -xplain1.3-6base-32k-step1647 Layer 31 | input_layernorm Layer 31 input_layernorm 4096 0.001660 signed -xplain1.3-6base-32k-step1647 Layer 33 | mlp.gate_proj Layer 33 mlp.gate_proj 52428800 0.001646 signed -xplain1.3-6base-32k-step1647 Layer 19 | input_layernorm Layer 19 input_layernorm 4096 0.001646 signed -xplain1.3-6base-32k-step1647 Layer 35 | self_attn.q_proj Layer 35 self_attn.q_proj 16777216 0.001643 signed -xplain1.3-6base-32k-step1647 Layer 34 | mlp.down_proj Layer 34 mlp.down_proj 52428800 0.001639 signed -xplain1.3-6base-32k-step1647 Layer 38 | self_attn.q_proj Layer 38 self_attn.q_proj 16777216 0.001634 signed -xplain1.3-6base-32k-step1647 Layer 39 | self_attn.q_proj Layer 39 self_attn.q_proj 16777216 0.001634 signed -xplain1.3-6base-32k-step1647 Layer 36 | mlp.down_proj Layer 36 mlp.down_proj 52428800 0.001630 signed -xplain1.3-6base-32k-step1647 Layer 34 | self_attn.q_proj Layer 34 self_attn.q_proj 16777216 0.001624 signed -xplain1.3-6base-32k-step1647 Layer 31 | mlp.gate_proj Layer 31 mlp.gate_proj 52428800 0.001623 signed -xplain1.3-6base-32k-step1647 Layer 32 | mlp.gate_proj Layer 32 mlp.gate_proj 52428800 0.001612 signed -xplain1.3-6base-32k-step1647 Layer 37 | mlp.down_proj Layer 37 mlp.down_proj 52428800 0.001598 signed -xplain1.3-6base-32k-step1647 Layer 30 | mlp.gate_proj Layer 30 mlp.gate_proj 52428800 0.001596 signed -xplain1.3-6base-32k-step1647 Layer 37 | self_attn.q_proj Layer 37 self_attn.q_proj 16777216 0.001596 signed -xplain1.3-6base-32k-step1647 Layer 29 | mlp.gate_proj Layer 29 mlp.gate_proj 52428800 0.001587 signed -xplain1.3-6base-32k-step1647 Layer 29 | mlp.down_proj Layer 29 mlp.down_proj 52428800 0.001586 signed -xplain1.3-6base-32k-step1647 Layer 28 | mlp.gate_proj Layer 28 mlp.gate_proj 52428800 0.001582 signed -xplain1.3-6base-32k-step1647 Layer 31 | mlp.down_proj Layer 31 mlp.down_proj 52428800 0.001582 signed -xplain1.3-6base-32k-step1647 Layer 33 | mlp.up_proj Layer 33 mlp.up_proj 52428800 0.001581 signed -xplain1.3-6base-32k-step1647 Layer 35 | mlp.down_proj Layer 35 mlp.down_proj 52428800 0.001580 signed -xplain1.3-6base-32k-step1647 Layer 30 | mlp.down_proj Layer 30 mlp.down_proj 52428800 0.001580 signed -xplain1.3-6base-32k-step1647 Layer 31 | mlp.up_proj Layer 31 mlp.up_proj 52428800 0.001579 signed -xplain1.3-6base-32k-step1647 Layer 28 | mlp.down_proj Layer 28 mlp.down_proj 52428800 0.001577 signed -xplain1.3-6base-32k-step1647 Layer 27 | mlp.gate_proj Layer 27 mlp.gate_proj 52428800 0.001575 signed + model sublayer block_label module_type count std stat_mode +xplain1.3-6base-32k-step1647 Layer 22 | input_layernorm Layer 22 input_layernorm 4096 0.005447 signed +xplain1.3-6base-32k-step1647 Layer 25 | input_layernorm Layer 25 input_layernorm 4096 0.005222 signed +xplain1.3-6base-32k-step1647 Layer 28 | input_layernorm Layer 28 input_layernorm 4096 0.005059 signed +xplain1.3-6base-32k-step1647 Layer 18 | input_layernorm Layer 18 input_layernorm 4096 0.003657 signed +xplain1.3-6base-32k-step1647 Layer 21 | input_layernorm Layer 21 input_layernorm 4096 0.003223 signed +xplain1.3-6base-32k-step1647 Layer 38 | input_layernorm Layer 38 input_layernorm 4096 0.002641 signed +xplain1.3-6base-32k-step1647 Layer 36 | input_layernorm Layer 36 input_layernorm 4096 0.001945 signed +xplain1.3-6base-32k-step1647 Layer 17 | input_layernorm Layer 17 input_layernorm 4096 0.001918 signed +xplain1.3-6base-32k-step1647 Layer 00 | input_layernorm Layer 00 input_layernorm 4096 0.001915 signed +xplain1.3-6base-32k-step1647 Layer 37 | mlp.gate_proj Layer 37 mlp.gate_proj 52428800 0.001854 signed +xplain1.3-6base-32k-step1647 Layer 37 | input_layernorm Layer 37 input_layernorm 4096 0.001846 signed +xplain1.3-6base-32k-step1647 Layer 38 | mlp.gate_proj Layer 38 mlp.gate_proj 52428800 0.001841 signed +xplain1.3-6base-32k-step1647 Layer 36 | mlp.gate_proj Layer 36 mlp.gate_proj 52428800 0.001839 signed +xplain1.3-6base-32k-step1647 Layer 35 | input_layernorm Layer 35 input_layernorm 4096 0.001798 signed +xplain1.3-6base-32k-step1647 Layer 37 | mlp.up_proj Layer 37 mlp.up_proj 52428800 0.001796 signed +xplain1.3-6base-32k-step1647 Layer 36 | mlp.up_proj Layer 36 mlp.up_proj 52428800 0.001786 signed +xplain1.3-6base-32k-step1647 Layer 34 | mlp.gate_proj Layer 34 mlp.gate_proj 52428800 0.001780 signed +xplain1.3-6base-32k-step1647 Layer 35 | mlp.gate_proj Layer 35 mlp.gate_proj 52428800 0.001771 signed +xplain1.3-6base-32k-step1647 Layer 38 | mlp.up_proj Layer 38 mlp.up_proj 52428800 0.001767 signed +xplain1.3-6base-32k-step1647 Layer 34 | mlp.up_proj Layer 34 mlp.up_proj 52428800 0.001721 signed +xplain1.3-6base-32k-step1647 Layer 35 | mlp.up_proj Layer 35 mlp.up_proj 52428800 0.001705 signed +xplain1.3-6base-32k-step1647 Layer 39 | mlp.gate_proj Layer 39 mlp.gate_proj 52428800 0.001701 signed +xplain1.3-6base-32k-step1647 Layer 36 | self_attn.q_proj Layer 36 self_attn.q_proj 16777216 0.001669 signed +xplain1.3-6base-32k-step1647 Layer 30 | input_layernorm Layer 30 input_layernorm 4096 0.001663 signed +xplain1.3-6base-32k-step1647 Layer 31 | input_layernorm Layer 31 input_layernorm 4096 0.001660 signed +xplain1.3-6base-32k-step1647 Layer 33 | mlp.gate_proj Layer 33 mlp.gate_proj 52428800 0.001646 signed +xplain1.3-6base-32k-step1647 Layer 19 | input_layernorm Layer 19 input_layernorm 4096 0.001646 signed +xplain1.3-6base-32k-step1647 Layer 35 | self_attn.q_proj Layer 35 self_attn.q_proj 16777216 0.001643 signed +xplain1.3-6base-32k-step1647 Layer 34 | mlp.down_proj Layer 34 mlp.down_proj 52428800 0.001639 signed +xplain1.3-6base-32k-step1647 Layer 38 | self_attn.q_proj Layer 38 self_attn.q_proj 16777216 0.001634 signed +xplain1.3-6base-32k-step1647 Layer 39 | self_attn.q_proj Layer 39 self_attn.q_proj 16777216 0.001634 signed +xplain1.3-6base-32k-step1647 Layer 36 | mlp.down_proj Layer 36 mlp.down_proj 52428800 0.001630 signed +xplain1.3-6base-32k-step1647 Layer 34 | self_attn.q_proj Layer 34 self_attn.q_proj 16777216 0.001624 signed +xplain1.3-6base-32k-step1647 Layer 31 | mlp.gate_proj Layer 31 mlp.gate_proj 52428800 0.001623 signed +xplain1.3-6base-32k-step1647 Layer 32 | mlp.gate_proj Layer 32 mlp.gate_proj 52428800 0.001612 signed +xplain1.3-6base-32k-step1647 Layer 37 | mlp.down_proj Layer 37 mlp.down_proj 52428800 0.001598 signed +xplain1.3-6base-32k-step1647 Layer 30 | mlp.gate_proj Layer 30 mlp.gate_proj 52428800 0.001596 signed +xplain1.3-6base-32k-step1647 Layer 37 | self_attn.q_proj Layer 37 self_attn.q_proj 16777216 0.001596 signed +xplain1.3-6base-32k-step1647 Layer 29 | mlp.gate_proj Layer 29 mlp.gate_proj 52428800 0.001587 signed +xplain1.3-6base-32k-step1647 Layer 29 | mlp.down_proj Layer 29 mlp.down_proj 52428800 0.001586 signed +xplain1.3-6base-32k-step1647 Layer 28 | mlp.gate_proj Layer 28 mlp.gate_proj 52428800 0.001582 signed +xplain1.3-6base-32k-step1647 Layer 31 | mlp.down_proj Layer 31 mlp.down_proj 52428800 0.001582 signed +xplain1.3-6base-32k-step1647 Layer 33 | mlp.up_proj Layer 33 mlp.up_proj 52428800 0.001581 signed +xplain1.3-6base-32k-step1647 Layer 35 | mlp.down_proj Layer 35 mlp.down_proj 52428800 0.001580 signed +xplain1.3-6base-32k-step1647 Layer 30 | mlp.down_proj Layer 30 mlp.down_proj 52428800 0.001580 signed +xplain1.3-6base-32k-step1647 Layer 31 | mlp.up_proj Layer 31 mlp.up_proj 52428800 0.001579 signed +xplain1.3-6base-32k-step1647 Layer 28 | mlp.down_proj Layer 28 mlp.down_proj 52428800 0.001577 signed +xplain1.3-6base-32k-step1647 Layer 27 | mlp.gate_proj Layer 27 mlp.gate_proj 52428800 0.001575 signed +xplain1.3-6base-32k-step1647 Layer 39 | mlp.up_proj Layer 39 mlp.up_proj 52428800 0.001573 signed +xplain1.3-6base-32k-step1647 Layer 21 | self_attn.q_proj Layer 21 self_attn.q_proj 16777216 0.001572 signed ### `xplain1.3-6base-64k-step1647` - model sublayer block_label module_type count std stat_mode -xplain1.3-6base-64k-step1647 Layer 22 | input_layernorm Layer 22 input_layernorm 4096 0.006970 signed -xplain1.3-6base-64k-step1647 Final Norm | model.norm.weight Final Norm final_norm 4096 0.006869 signed -xplain1.3-6base-64k-step1647 Layer 25 | input_layernorm Layer 25 input_layernorm 4096 0.006613 signed -xplain1.3-6base-64k-step1647 Layer 28 | input_layernorm Layer 28 input_layernorm 4096 0.006396 signed -xplain1.3-6base-64k-step1647 Layer 18 | input_layernorm Layer 18 input_layernorm 4096 0.004488 signed -xplain1.3-6base-64k-step1647 Layer 21 | input_layernorm Layer 21 input_layernorm 4096 0.004040 signed -xplain1.3-6base-64k-step1647 Layer 38 | input_layernorm Layer 38 input_layernorm 4096 0.003279 signed -xplain1.3-6base-64k-step1647 Layer 00 | input_layernorm Layer 00 input_layernorm 4096 0.002411 signed -xplain1.3-6base-64k-step1647 Layer 36 | input_layernorm Layer 36 input_layernorm 4096 0.002349 signed -xplain1.3-6base-64k-step1647 Layer 37 | input_layernorm Layer 37 input_layernorm 4096 0.002266 signed -xplain1.3-6base-64k-step1647 Layer 17 | input_layernorm Layer 17 input_layernorm 4096 0.002246 signed -xplain1.3-6base-64k-step1647 Layer 35 | input_layernorm Layer 35 input_layernorm 4096 0.002107 signed -xplain1.3-6base-64k-step1647 Layer 37 | mlp.gate_proj Layer 37 mlp.gate_proj 52428800 0.002046 signed -xplain1.3-6base-64k-step1647 Layer 38 | mlp.gate_proj Layer 38 mlp.gate_proj 52428800 0.002033 signed -xplain1.3-6base-64k-step1647 Layer 36 | mlp.gate_proj Layer 36 mlp.gate_proj 52428800 0.002031 signed -xplain1.3-6base-64k-step1647 Embedding | model.embed_tokens.weight Embedding embed_tokens 201326592 0.002014 signed -xplain1.3-6base-64k-step1647 Layer 37 | mlp.up_proj Layer 37 mlp.up_proj 52428800 0.001987 signed -xplain1.3-6base-64k-step1647 Layer 36 | mlp.up_proj Layer 36 mlp.up_proj 52428800 0.001982 signed -xplain1.3-6base-64k-step1647 Layer 19 | input_layernorm Layer 19 input_layernorm 4096 0.001978 signed -xplain1.3-6base-64k-step1647 Layer 34 | mlp.gate_proj Layer 34 mlp.gate_proj 52428800 0.001977 signed -xplain1.3-6base-64k-step1647 Layer 35 | mlp.gate_proj Layer 35 mlp.gate_proj 52428800 0.001965 signed -xplain1.3-6base-64k-step1647 Layer 30 | input_layernorm Layer 30 input_layernorm 4096 0.001959 signed -xplain1.3-6base-64k-step1647 Layer 38 | mlp.up_proj Layer 38 mlp.up_proj 52428800 0.001954 signed -xplain1.3-6base-64k-step1647 Layer 34 | mlp.up_proj Layer 34 mlp.up_proj 52428800 0.001919 signed -xplain1.3-6base-64k-step1647 Layer 31 | input_layernorm Layer 31 input_layernorm 4096 0.001907 signed -xplain1.3-6base-64k-step1647 Layer 35 | mlp.up_proj Layer 35 mlp.up_proj 52428800 0.001898 signed -xplain1.3-6base-64k-step1647 Layer 39 | mlp.gate_proj Layer 39 mlp.gate_proj 52428800 0.001897 signed -xplain1.3-6base-64k-step1647 Layer 14 | input_layernorm Layer 14 input_layernorm 4096 0.001861 signed -xplain1.3-6base-64k-step1647 Layer 33 | mlp.gate_proj Layer 33 mlp.gate_proj 52428800 0.001850 signed -xplain1.3-6base-64k-step1647 Layer 36 | self_attn.q_proj Layer 36 self_attn.q_proj 16777216 0.001844 signed -xplain1.3-6base-64k-step1647 Layer 34 | mlp.down_proj Layer 34 mlp.down_proj 52428800 0.001831 signed -xplain1.3-6base-64k-step1647 Layer 35 | self_attn.q_proj Layer 35 self_attn.q_proj 16777216 0.001824 signed -xplain1.3-6base-64k-step1647 Layer 31 | mlp.gate_proj Layer 31 mlp.gate_proj 52428800 0.001822 signed -xplain1.3-6base-64k-step1647 Layer 36 | mlp.down_proj Layer 36 mlp.down_proj 52428800 0.001817 signed -xplain1.3-6base-64k-step1647 Layer 34 | self_attn.q_proj Layer 34 self_attn.q_proj 16777216 0.001816 signed -xplain1.3-6base-64k-step1647 Layer 32 | mlp.gate_proj Layer 32 mlp.gate_proj 52428800 0.001813 signed -xplain1.3-6base-64k-step1647 Layer 39 | self_attn.q_proj Layer 39 self_attn.q_proj 16777216 0.001804 signed -xplain1.3-6base-64k-step1647 Layer 32 | input_layernorm Layer 32 input_layernorm 4096 0.001804 signed -xplain1.3-6base-64k-step1647 Layer 33 | input_layernorm Layer 33 input_layernorm 4096 0.001803 signed -xplain1.3-6base-64k-step1647 Layer 38 | self_attn.q_proj Layer 38 self_attn.q_proj 16777216 0.001798 signed -xplain1.3-6base-64k-step1647 Layer 30 | mlp.gate_proj Layer 30 mlp.gate_proj 52428800 0.001794 signed -xplain1.3-6base-64k-step1647 Layer 29 | mlp.down_proj Layer 29 mlp.down_proj 52428800 0.001790 signed -xplain1.3-6base-64k-step1647 Layer 31 | mlp.down_proj Layer 31 mlp.down_proj 52428800 0.001785 signed -xplain1.3-6base-64k-step1647 Layer 30 | mlp.down_proj Layer 30 mlp.down_proj 52428800 0.001785 signed -xplain1.3-6base-64k-step1647 Layer 33 | mlp.up_proj Layer 33 mlp.up_proj 52428800 0.001784 signed -xplain1.3-6base-64k-step1647 Layer 15 | input_layernorm Layer 15 input_layernorm 4096 0.001783 signed -xplain1.3-6base-64k-step1647 Layer 29 | mlp.gate_proj Layer 29 mlp.gate_proj 52428800 0.001782 signed -xplain1.3-6base-64k-step1647 Layer 31 | mlp.up_proj Layer 31 mlp.up_proj 52428800 0.001781 signed -xplain1.3-6base-64k-step1647 Layer 28 | mlp.down_proj Layer 28 mlp.down_proj 52428800 0.001779 signed -xplain1.3-6base-64k-step1647 Layer 28 | mlp.gate_proj Layer 28 mlp.gate_proj 52428800 0.001775 signed + model sublayer block_label module_type count std stat_mode +xplain1.3-6base-64k-step1647 Layer 22 | input_layernorm Layer 22 input_layernorm 4096 0.006970 signed +xplain1.3-6base-64k-step1647 Layer 25 | input_layernorm Layer 25 input_layernorm 4096 0.006613 signed +xplain1.3-6base-64k-step1647 Layer 28 | input_layernorm Layer 28 input_layernorm 4096 0.006396 signed +xplain1.3-6base-64k-step1647 Layer 18 | input_layernorm Layer 18 input_layernorm 4096 0.004488 signed +xplain1.3-6base-64k-step1647 Layer 21 | input_layernorm Layer 21 input_layernorm 4096 0.004040 signed +xplain1.3-6base-64k-step1647 Layer 38 | input_layernorm Layer 38 input_layernorm 4096 0.003279 signed +xplain1.3-6base-64k-step1647 Layer 00 | input_layernorm Layer 00 input_layernorm 4096 0.002411 signed +xplain1.3-6base-64k-step1647 Layer 36 | input_layernorm Layer 36 input_layernorm 4096 0.002349 signed +xplain1.3-6base-64k-step1647 Layer 37 | input_layernorm Layer 37 input_layernorm 4096 0.002266 signed +xplain1.3-6base-64k-step1647 Layer 17 | input_layernorm Layer 17 input_layernorm 4096 0.002246 signed +xplain1.3-6base-64k-step1647 Layer 35 | input_layernorm Layer 35 input_layernorm 4096 0.002107 signed +xplain1.3-6base-64k-step1647 Layer 37 | mlp.gate_proj Layer 37 mlp.gate_proj 52428800 0.002046 signed +xplain1.3-6base-64k-step1647 Layer 38 | mlp.gate_proj Layer 38 mlp.gate_proj 52428800 0.002033 signed +xplain1.3-6base-64k-step1647 Layer 36 | mlp.gate_proj Layer 36 mlp.gate_proj 52428800 0.002031 signed +xplain1.3-6base-64k-step1647 Layer 37 | mlp.up_proj Layer 37 mlp.up_proj 52428800 0.001987 signed +xplain1.3-6base-64k-step1647 Layer 36 | mlp.up_proj Layer 36 mlp.up_proj 52428800 0.001982 signed +xplain1.3-6base-64k-step1647 Layer 19 | input_layernorm Layer 19 input_layernorm 4096 0.001978 signed +xplain1.3-6base-64k-step1647 Layer 34 | mlp.gate_proj Layer 34 mlp.gate_proj 52428800 0.001977 signed +xplain1.3-6base-64k-step1647 Layer 35 | mlp.gate_proj Layer 35 mlp.gate_proj 52428800 0.001965 signed +xplain1.3-6base-64k-step1647 Layer 30 | input_layernorm Layer 30 input_layernorm 4096 0.001959 signed +xplain1.3-6base-64k-step1647 Layer 38 | mlp.up_proj Layer 38 mlp.up_proj 52428800 0.001954 signed +xplain1.3-6base-64k-step1647 Layer 34 | mlp.up_proj Layer 34 mlp.up_proj 52428800 0.001919 signed +xplain1.3-6base-64k-step1647 Layer 31 | input_layernorm Layer 31 input_layernorm 4096 0.001907 signed +xplain1.3-6base-64k-step1647 Layer 35 | mlp.up_proj Layer 35 mlp.up_proj 52428800 0.001898 signed +xplain1.3-6base-64k-step1647 Layer 39 | mlp.gate_proj Layer 39 mlp.gate_proj 52428800 0.001897 signed +xplain1.3-6base-64k-step1647 Layer 14 | input_layernorm Layer 14 input_layernorm 4096 0.001861 signed +xplain1.3-6base-64k-step1647 Layer 33 | mlp.gate_proj Layer 33 mlp.gate_proj 52428800 0.001850 signed +xplain1.3-6base-64k-step1647 Layer 36 | self_attn.q_proj Layer 36 self_attn.q_proj 16777216 0.001844 signed +xplain1.3-6base-64k-step1647 Layer 34 | mlp.down_proj Layer 34 mlp.down_proj 52428800 0.001831 signed +xplain1.3-6base-64k-step1647 Layer 35 | self_attn.q_proj Layer 35 self_attn.q_proj 16777216 0.001824 signed +xplain1.3-6base-64k-step1647 Layer 31 | mlp.gate_proj Layer 31 mlp.gate_proj 52428800 0.001822 signed +xplain1.3-6base-64k-step1647 Layer 36 | mlp.down_proj Layer 36 mlp.down_proj 52428800 0.001817 signed +xplain1.3-6base-64k-step1647 Layer 34 | self_attn.q_proj Layer 34 self_attn.q_proj 16777216 0.001816 signed +xplain1.3-6base-64k-step1647 Layer 32 | mlp.gate_proj Layer 32 mlp.gate_proj 52428800 0.001813 signed +xplain1.3-6base-64k-step1647 Layer 39 | self_attn.q_proj Layer 39 self_attn.q_proj 16777216 0.001804 signed +xplain1.3-6base-64k-step1647 Layer 32 | input_layernorm Layer 32 input_layernorm 4096 0.001804 signed +xplain1.3-6base-64k-step1647 Layer 33 | input_layernorm Layer 33 input_layernorm 4096 0.001803 signed +xplain1.3-6base-64k-step1647 Layer 38 | self_attn.q_proj Layer 38 self_attn.q_proj 16777216 0.001798 signed +xplain1.3-6base-64k-step1647 Layer 30 | mlp.gate_proj Layer 30 mlp.gate_proj 52428800 0.001794 signed +xplain1.3-6base-64k-step1647 Layer 29 | mlp.down_proj Layer 29 mlp.down_proj 52428800 0.001790 signed +xplain1.3-6base-64k-step1647 Layer 31 | mlp.down_proj Layer 31 mlp.down_proj 52428800 0.001785 signed +xplain1.3-6base-64k-step1647 Layer 30 | mlp.down_proj Layer 30 mlp.down_proj 52428800 0.001785 signed +xplain1.3-6base-64k-step1647 Layer 33 | mlp.up_proj Layer 33 mlp.up_proj 52428800 0.001784 signed +xplain1.3-6base-64k-step1647 Layer 15 | input_layernorm Layer 15 input_layernorm 4096 0.001783 signed +xplain1.3-6base-64k-step1647 Layer 29 | mlp.gate_proj Layer 29 mlp.gate_proj 52428800 0.001782 signed +xplain1.3-6base-64k-step1647 Layer 31 | mlp.up_proj Layer 31 mlp.up_proj 52428800 0.001781 signed +xplain1.3-6base-64k-step1647 Layer 28 | mlp.down_proj Layer 28 mlp.down_proj 52428800 0.001779 signed +xplain1.3-6base-64k-step1647 Layer 28 | mlp.gate_proj Layer 28 mlp.gate_proj 52428800 0.001775 signed +xplain1.3-6base-64k-step1647 Layer 37 | mlp.down_proj Layer 37 mlp.down_proj 52428800 0.001774 signed +xplain1.3-6base-64k-step1647 Layer 27 | mlp.down_proj Layer 27 mlp.down_proj 52428800 0.001766 signed ### `xplain1.3-6base-8k-step2199` - model sublayer block_label module_type count std stat_mode -xplain1.3-6base-8k-step2199 Final Norm | model.norm.weight Final Norm final_norm 4096 0.002388 signed -xplain1.3-6base-8k-step2199 Layer 22 | input_layernorm Layer 22 input_layernorm 4096 0.002092 signed -xplain1.3-6base-8k-step2199 Layer 25 | input_layernorm Layer 25 input_layernorm 4096 0.002092 signed -xplain1.3-6base-8k-step2199 Layer 28 | input_layernorm Layer 28 input_layernorm 4096 0.002019 signed -xplain1.3-6base-8k-step2199 Layer 18 | input_layernorm Layer 18 input_layernorm 4096 0.001531 signed -xplain1.3-6base-8k-step2199 Layer 21 | input_layernorm Layer 21 input_layernorm 4096 0.001351 signed -xplain1.3-6base-8k-step2199 Layer 37 | mlp.gate_proj Layer 37 mlp.gate_proj 52428800 0.001253 signed -xplain1.3-6base-8k-step2199 Layer 38 | mlp.gate_proj Layer 38 mlp.gate_proj 52428800 0.001244 signed -xplain1.3-6base-8k-step2199 Layer 36 | mlp.gate_proj Layer 36 mlp.gate_proj 52428800 0.001238 signed -xplain1.3-6base-8k-step2199 Layer 37 | mlp.up_proj Layer 37 mlp.up_proj 52428800 0.001196 signed -xplain1.3-6base-8k-step2199 Layer 38 | mlp.up_proj Layer 38 mlp.up_proj 52428800 0.001177 signed -xplain1.3-6base-8k-step2199 Layer 34 | mlp.gate_proj Layer 34 mlp.gate_proj 52428800 0.001174 signed -xplain1.3-6base-8k-step2199 Layer 36 | mlp.up_proj Layer 36 mlp.up_proj 52428800 0.001173 signed -xplain1.3-6base-8k-step2199 Layer 35 | mlp.gate_proj Layer 35 mlp.gate_proj 52428800 0.001170 signed -xplain1.3-6base-8k-step2199 Layer 38 | input_layernorm Layer 38 input_layernorm 4096 0.001120 signed -xplain1.3-6base-8k-step2199 Layer 34 | mlp.up_proj Layer 34 mlp.up_proj 52428800 0.001119 signed -xplain1.3-6base-8k-step2199 Layer 36 | self_attn.q_proj Layer 36 self_attn.q_proj 16777216 0.001116 signed -xplain1.3-6base-8k-step2199 Layer 39 | mlp.gate_proj Layer 39 mlp.gate_proj 52428800 0.001115 signed -xplain1.3-6base-8k-step2199 Layer 35 | mlp.up_proj Layer 35 mlp.up_proj 52428800 0.001107 signed -xplain1.3-6base-8k-step2199 Layer 38 | self_attn.q_proj Layer 38 self_attn.q_proj 16777216 0.001101 signed -xplain1.3-6base-8k-step2199 Layer 39 | self_attn.q_proj Layer 39 self_attn.q_proj 16777216 0.001095 signed -xplain1.3-6base-8k-step2199 Layer 17 | input_layernorm Layer 17 input_layernorm 4096 0.001087 signed -xplain1.3-6base-8k-step2199 Layer 35 | self_attn.q_proj Layer 35 self_attn.q_proj 16777216 0.001075 signed -xplain1.3-6base-8k-step2199 Layer 34 | mlp.down_proj Layer 34 mlp.down_proj 52428800 0.001072 signed -xplain1.3-6base-8k-step2199 Layer 36 | mlp.down_proj Layer 36 mlp.down_proj 52428800 0.001070 signed -xplain1.3-6base-8k-step2199 Layer 37 | mlp.down_proj Layer 37 mlp.down_proj 52428800 0.001064 signed -xplain1.3-6base-8k-step2199 Layer 37 | self_attn.q_proj Layer 37 self_attn.q_proj 16777216 0.001064 signed -xplain1.3-6base-8k-step2199 Layer 38 | mlp.down_proj Layer 38 mlp.down_proj 52428800 0.001052 signed -xplain1.3-6base-8k-step2199 Layer 36 | self_attn.k_proj Layer 36 self_attn.k_proj 4194304 0.001038 signed -xplain1.3-6base-8k-step2199 Layer 35 | mlp.down_proj Layer 35 mlp.down_proj 52428800 0.001028 signed -xplain1.3-6base-8k-step2199 Embedding | model.embed_tokens.weight Embedding embed_tokens 201326592 0.001027 signed -xplain1.3-6base-8k-step2199 Layer 33 | mlp.gate_proj Layer 33 mlp.gate_proj 52428800 0.001019 signed -xplain1.3-6base-8k-step2199 Layer 34 | self_attn.q_proj Layer 34 self_attn.q_proj 16777216 0.001015 signed -xplain1.3-6base-8k-step2199 Layer 39 | mlp.up_proj Layer 39 mlp.up_proj 52428800 0.001007 signed -xplain1.3-6base-8k-step2199 Layer 39 | self_attn.k_proj Layer 39 self_attn.k_proj 4194304 0.001005 signed -xplain1.3-6base-8k-step2199 Layer 31 | mlp.gate_proj Layer 31 mlp.gate_proj 52428800 0.000999 signed -xplain1.3-6base-8k-step2199 Layer 35 | self_attn.k_proj Layer 35 self_attn.k_proj 4194304 0.000989 signed -xplain1.3-6base-8k-step2199 Layer 38 | self_attn.k_proj Layer 38 self_attn.k_proj 4194304 0.000988 signed -xplain1.3-6base-8k-step2199 Layer 32 | mlp.gate_proj Layer 32 mlp.gate_proj 52428800 0.000984 signed -xplain1.3-6base-8k-step2199 Layer 35 | input_layernorm Layer 35 input_layernorm 4096 0.000981 signed -xplain1.3-6base-8k-step2199 Layer 21 | self_attn.q_proj Layer 21 self_attn.q_proj 16777216 0.000980 signed -xplain1.3-6base-8k-step2199 Layer 17 | self_attn.q_proj Layer 17 self_attn.q_proj 16777216 0.000976 signed -xplain1.3-6base-8k-step2199 Layer 28 | mlp.gate_proj Layer 28 mlp.gate_proj 52428800 0.000975 signed -xplain1.3-6base-8k-step2199 Layer 30 | mlp.gate_proj Layer 30 mlp.gate_proj 52428800 0.000974 signed -xplain1.3-6base-8k-step2199 Layer 17 | mlp.gate_proj Layer 17 mlp.gate_proj 52428800 0.000974 signed -xplain1.3-6base-8k-step2199 Layer 29 | mlp.gate_proj Layer 29 mlp.gate_proj 52428800 0.000974 signed -xplain1.3-6base-8k-step2199 Layer 27 | mlp.gate_proj Layer 27 mlp.gate_proj 52428800 0.000974 signed -xplain1.3-6base-8k-step2199 Layer 18 | mlp.gate_proj Layer 18 mlp.gate_proj 52428800 0.000973 signed -xplain1.3-6base-8k-step2199 Layer 19 | self_attn.q_proj Layer 19 self_attn.q_proj 16777216 0.000971 signed -xplain1.3-6base-8k-step2199 Layer 19 | mlp.gate_proj Layer 19 mlp.gate_proj 52428800 0.000970 signed + model sublayer block_label module_type count std stat_mode +xplain1.3-6base-8k-step2199 Layer 22 | input_layernorm Layer 22 input_layernorm 4096 0.002092 signed +xplain1.3-6base-8k-step2199 Layer 25 | input_layernorm Layer 25 input_layernorm 4096 0.002092 signed +xplain1.3-6base-8k-step2199 Layer 28 | input_layernorm Layer 28 input_layernorm 4096 0.002019 signed +xplain1.3-6base-8k-step2199 Layer 18 | input_layernorm Layer 18 input_layernorm 4096 0.001531 signed +xplain1.3-6base-8k-step2199 Layer 21 | input_layernorm Layer 21 input_layernorm 4096 0.001351 signed +xplain1.3-6base-8k-step2199 Layer 37 | mlp.gate_proj Layer 37 mlp.gate_proj 52428800 0.001253 signed +xplain1.3-6base-8k-step2199 Layer 38 | mlp.gate_proj Layer 38 mlp.gate_proj 52428800 0.001244 signed +xplain1.3-6base-8k-step2199 Layer 36 | mlp.gate_proj Layer 36 mlp.gate_proj 52428800 0.001238 signed +xplain1.3-6base-8k-step2199 Layer 37 | mlp.up_proj Layer 37 mlp.up_proj 52428800 0.001196 signed +xplain1.3-6base-8k-step2199 Layer 38 | mlp.up_proj Layer 38 mlp.up_proj 52428800 0.001177 signed +xplain1.3-6base-8k-step2199 Layer 34 | mlp.gate_proj Layer 34 mlp.gate_proj 52428800 0.001174 signed +xplain1.3-6base-8k-step2199 Layer 36 | mlp.up_proj Layer 36 mlp.up_proj 52428800 0.001173 signed +xplain1.3-6base-8k-step2199 Layer 35 | mlp.gate_proj Layer 35 mlp.gate_proj 52428800 0.001170 signed +xplain1.3-6base-8k-step2199 Layer 38 | input_layernorm Layer 38 input_layernorm 4096 0.001120 signed +xplain1.3-6base-8k-step2199 Layer 34 | mlp.up_proj Layer 34 mlp.up_proj 52428800 0.001119 signed +xplain1.3-6base-8k-step2199 Layer 36 | self_attn.q_proj Layer 36 self_attn.q_proj 16777216 0.001116 signed +xplain1.3-6base-8k-step2199 Layer 39 | mlp.gate_proj Layer 39 mlp.gate_proj 52428800 0.001115 signed +xplain1.3-6base-8k-step2199 Layer 35 | mlp.up_proj Layer 35 mlp.up_proj 52428800 0.001107 signed +xplain1.3-6base-8k-step2199 Layer 38 | self_attn.q_proj Layer 38 self_attn.q_proj 16777216 0.001101 signed +xplain1.3-6base-8k-step2199 Layer 39 | self_attn.q_proj Layer 39 self_attn.q_proj 16777216 0.001095 signed +xplain1.3-6base-8k-step2199 Layer 17 | input_layernorm Layer 17 input_layernorm 4096 0.001087 signed +xplain1.3-6base-8k-step2199 Layer 35 | self_attn.q_proj Layer 35 self_attn.q_proj 16777216 0.001075 signed +xplain1.3-6base-8k-step2199 Layer 34 | mlp.down_proj Layer 34 mlp.down_proj 52428800 0.001072 signed +xplain1.3-6base-8k-step2199 Layer 36 | mlp.down_proj Layer 36 mlp.down_proj 52428800 0.001070 signed +xplain1.3-6base-8k-step2199 Layer 37 | mlp.down_proj Layer 37 mlp.down_proj 52428800 0.001064 signed +xplain1.3-6base-8k-step2199 Layer 37 | self_attn.q_proj Layer 37 self_attn.q_proj 16777216 0.001064 signed +xplain1.3-6base-8k-step2199 Layer 38 | mlp.down_proj Layer 38 mlp.down_proj 52428800 0.001052 signed +xplain1.3-6base-8k-step2199 Layer 36 | self_attn.k_proj Layer 36 self_attn.k_proj 4194304 0.001038 signed +xplain1.3-6base-8k-step2199 Layer 35 | mlp.down_proj Layer 35 mlp.down_proj 52428800 0.001028 signed +xplain1.3-6base-8k-step2199 Layer 33 | mlp.gate_proj Layer 33 mlp.gate_proj 52428800 0.001019 signed +xplain1.3-6base-8k-step2199 Layer 34 | self_attn.q_proj Layer 34 self_attn.q_proj 16777216 0.001015 signed +xplain1.3-6base-8k-step2199 Layer 39 | mlp.up_proj Layer 39 mlp.up_proj 52428800 0.001007 signed +xplain1.3-6base-8k-step2199 Layer 39 | self_attn.k_proj Layer 39 self_attn.k_proj 4194304 0.001005 signed +xplain1.3-6base-8k-step2199 Layer 31 | mlp.gate_proj Layer 31 mlp.gate_proj 52428800 0.000999 signed +xplain1.3-6base-8k-step2199 Layer 35 | self_attn.k_proj Layer 35 self_attn.k_proj 4194304 0.000989 signed +xplain1.3-6base-8k-step2199 Layer 38 | self_attn.k_proj Layer 38 self_attn.k_proj 4194304 0.000988 signed +xplain1.3-6base-8k-step2199 Layer 32 | mlp.gate_proj Layer 32 mlp.gate_proj 52428800 0.000984 signed +xplain1.3-6base-8k-step2199 Layer 35 | input_layernorm Layer 35 input_layernorm 4096 0.000981 signed +xplain1.3-6base-8k-step2199 Layer 21 | self_attn.q_proj Layer 21 self_attn.q_proj 16777216 0.000980 signed +xplain1.3-6base-8k-step2199 Layer 17 | self_attn.q_proj Layer 17 self_attn.q_proj 16777216 0.000976 signed +xplain1.3-6base-8k-step2199 Layer 28 | mlp.gate_proj Layer 28 mlp.gate_proj 52428800 0.000975 signed +xplain1.3-6base-8k-step2199 Layer 30 | mlp.gate_proj Layer 30 mlp.gate_proj 52428800 0.000974 signed +xplain1.3-6base-8k-step2199 Layer 17 | mlp.gate_proj Layer 17 mlp.gate_proj 52428800 0.000974 signed +xplain1.3-6base-8k-step2199 Layer 29 | mlp.gate_proj Layer 29 mlp.gate_proj 52428800 0.000974 signed +xplain1.3-6base-8k-step2199 Layer 27 | mlp.gate_proj Layer 27 mlp.gate_proj 52428800 0.000974 signed +xplain1.3-6base-8k-step2199 Layer 18 | mlp.gate_proj Layer 18 mlp.gate_proj 52428800 0.000973 signed +xplain1.3-6base-8k-step2199 Layer 19 | self_attn.q_proj Layer 19 self_attn.q_proj 16777216 0.000971 signed +xplain1.3-6base-8k-step2199 Layer 19 | mlp.gate_proj Layer 19 mlp.gate_proj 52428800 0.000970 signed +xplain1.3-6base-8k-step2199 Layer 26 | mlp.gate_proj Layer 26 mlp.gate_proj 52428800 0.000969 signed +xplain1.3-6base-8k-step2199 Layer 31 | mlp.down_proj Layer 31 mlp.down_proj 52428800 0.000967 signed ### `xplain1.4-6base-16k-step2319` - model sublayer block_label module_type count std stat_mode -xplain1.4-6base-16k-step2319 Final Norm | model.norm.weight Final Norm final_norm 4096 0.005594 signed -xplain1.4-6base-16k-step2319 Layer 22 | input_layernorm Layer 22 input_layernorm 4096 0.004097 signed -xplain1.4-6base-16k-step2319 Layer 28 | input_layernorm Layer 28 input_layernorm 4096 0.004061 signed -xplain1.4-6base-16k-step2319 Layer 25 | input_layernorm Layer 25 input_layernorm 4096 0.003967 signed -xplain1.4-6base-16k-step2319 Layer 18 | input_layernorm Layer 18 input_layernorm 4096 0.003069 signed -xplain1.4-6base-16k-step2319 Layer 21 | input_layernorm Layer 21 input_layernorm 4096 0.002584 signed -xplain1.4-6base-16k-step2319 Layer 38 | input_layernorm Layer 38 input_layernorm 4096 0.002278 signed -xplain1.4-6base-16k-step2319 Layer 38 | mlp.gate_proj Layer 38 mlp.gate_proj 52428800 0.001990 signed -xplain1.4-6base-16k-step2319 Layer 37 | mlp.gate_proj Layer 37 mlp.gate_proj 52428800 0.001911 signed -xplain1.4-6base-16k-step2319 Layer 38 | mlp.up_proj Layer 38 mlp.up_proj 52428800 0.001874 signed -xplain1.4-6base-16k-step2319 Layer 39 | mlp.gate_proj Layer 39 mlp.gate_proj 52428800 0.001832 signed -xplain1.4-6base-16k-step2319 Layer 37 | mlp.up_proj Layer 37 mlp.up_proj 52428800 0.001829 signed -xplain1.4-6base-16k-step2319 Layer 38 | self_attn.q_proj Layer 38 self_attn.q_proj 16777216 0.001743 signed -xplain1.4-6base-16k-step2319 Layer 39 | post_attention_layernorm Layer 39 post_attention_layernorm 4096 0.001732 signed -xplain1.4-6base-16k-step2319 Layer 39 | self_attn.q_proj Layer 39 self_attn.q_proj 16777216 0.001715 signed -xplain1.4-6base-16k-step2319 Layer 38 | mlp.down_proj Layer 38 mlp.down_proj 52428800 0.001702 signed -xplain1.4-6base-16k-step2319 Embedding | model.embed_tokens.weight Embedding embed_tokens 201326592 0.001679 signed -xplain1.4-6base-16k-step2319 Layer 39 | mlp.up_proj Layer 39 mlp.up_proj 52428800 0.001678 signed -xplain1.4-6base-16k-step2319 Layer 36 | input_layernorm Layer 36 input_layernorm 4096 0.001660 signed -xplain1.4-6base-16k-step2319 Layer 37 | mlp.down_proj Layer 37 mlp.down_proj 52428800 0.001632 signed -xplain1.4-6base-16k-step2319 Layer 36 | mlp.gate_proj Layer 36 mlp.gate_proj 52428800 0.001611 signed -xplain1.4-6base-16k-step2319 Layer 37 | post_attention_layernorm Layer 37 post_attention_layernorm 4096 0.001591 signed -xplain1.4-6base-16k-step2319 Layer 37 | self_attn.q_proj Layer 37 self_attn.q_proj 16777216 0.001560 signed -xplain1.4-6base-16k-step2319 Layer 36 | mlp.up_proj Layer 36 mlp.up_proj 52428800 0.001555 signed -xplain1.4-6base-16k-step2319 Layer 35 | mlp.gate_proj Layer 35 mlp.gate_proj 52428800 0.001549 signed -xplain1.4-6base-16k-step2319 Layer 37 | input_layernorm Layer 37 input_layernorm 4096 0.001547 signed -xplain1.4-6base-16k-step2319 Layer 39 | self_attn.k_proj Layer 39 self_attn.k_proj 4194304 0.001546 signed -xplain1.4-6base-16k-step2319 Layer 38 | post_attention_layernorm Layer 38 post_attention_layernorm 4096 0.001541 signed -xplain1.4-6base-16k-step2319 Layer 38 | self_attn.k_proj Layer 38 self_attn.k_proj 4194304 0.001533 signed -xplain1.4-6base-16k-step2319 Layer 00 | input_layernorm Layer 00 input_layernorm 4096 0.001527 signed -xplain1.4-6base-16k-step2319 Layer 34 | mlp.gate_proj Layer 34 mlp.gate_proj 52428800 0.001513 signed -xplain1.4-6base-16k-step2319 Layer 35 | input_layernorm Layer 35 input_layernorm 4096 0.001508 signed -xplain1.4-6base-16k-step2319 Layer 30 | input_layernorm Layer 30 input_layernorm 4096 0.001490 signed -xplain1.4-6base-16k-step2319 Layer 17 | input_layernorm Layer 17 input_layernorm 4096 0.001487 signed -xplain1.4-6base-16k-step2319 Layer 35 | mlp.up_proj Layer 35 mlp.up_proj 52428800 0.001487 signed -xplain1.4-6base-16k-step2319 Layer 34 | mlp.up_proj Layer 34 mlp.up_proj 52428800 0.001458 signed -xplain1.4-6base-16k-step2319 Layer 36 | self_attn.q_proj Layer 36 self_attn.q_proj 16777216 0.001450 signed -xplain1.4-6base-16k-step2319 Layer 33 | mlp.gate_proj Layer 33 mlp.gate_proj 52428800 0.001444 signed -xplain1.4-6base-16k-step2319 Layer 36 | mlp.down_proj Layer 36 mlp.down_proj 52428800 0.001431 signed -xplain1.4-6base-16k-step2319 Layer 34 | self_attn.q_proj Layer 34 self_attn.q_proj 16777216 0.001430 signed -xplain1.4-6base-16k-step2319 Layer 39 | input_layernorm Layer 39 input_layernorm 4096 0.001430 signed -xplain1.4-6base-16k-step2319 Layer 31 | input_layernorm Layer 31 input_layernorm 4096 0.001421 signed -xplain1.4-6base-16k-step2319 Layer 32 | mlp.gate_proj Layer 32 mlp.gate_proj 52428800 0.001418 signed -xplain1.4-6base-16k-step2319 Layer 37 | self_attn.k_proj Layer 37 self_attn.k_proj 4194304 0.001413 signed -xplain1.4-6base-16k-step2319 Layer 31 | mlp.gate_proj Layer 31 mlp.gate_proj 52428800 0.001409 signed -xplain1.4-6base-16k-step2319 Layer 33 | mlp.up_proj Layer 33 mlp.up_proj 52428800 0.001392 signed -xplain1.4-6base-16k-step2319 Layer 34 | mlp.down_proj Layer 34 mlp.down_proj 52428800 0.001389 signed -xplain1.4-6base-16k-step2319 Layer 35 | self_attn.q_proj Layer 35 self_attn.q_proj 16777216 0.001387 signed -xplain1.4-6base-16k-step2319 Layer 30 | mlp.gate_proj Layer 30 mlp.gate_proj 52428800 0.001387 signed -xplain1.4-6base-16k-step2319 Layer 35 | mlp.down_proj Layer 35 mlp.down_proj 52428800 0.001381 signed + model sublayer block_label module_type count std stat_mode +xplain1.4-6base-16k-step2319 Layer 22 | input_layernorm Layer 22 input_layernorm 4096 0.004097 signed +xplain1.4-6base-16k-step2319 Layer 28 | input_layernorm Layer 28 input_layernorm 4096 0.004061 signed +xplain1.4-6base-16k-step2319 Layer 25 | input_layernorm Layer 25 input_layernorm 4096 0.003967 signed +xplain1.4-6base-16k-step2319 Layer 18 | input_layernorm Layer 18 input_layernorm 4096 0.003069 signed +xplain1.4-6base-16k-step2319 Layer 21 | input_layernorm Layer 21 input_layernorm 4096 0.002584 signed +xplain1.4-6base-16k-step2319 Layer 38 | input_layernorm Layer 38 input_layernorm 4096 0.002278 signed +xplain1.4-6base-16k-step2319 Layer 38 | mlp.gate_proj Layer 38 mlp.gate_proj 52428800 0.001990 signed +xplain1.4-6base-16k-step2319 Layer 37 | mlp.gate_proj Layer 37 mlp.gate_proj 52428800 0.001911 signed +xplain1.4-6base-16k-step2319 Layer 38 | mlp.up_proj Layer 38 mlp.up_proj 52428800 0.001874 signed +xplain1.4-6base-16k-step2319 Layer 39 | mlp.gate_proj Layer 39 mlp.gate_proj 52428800 0.001832 signed +xplain1.4-6base-16k-step2319 Layer 37 | mlp.up_proj Layer 37 mlp.up_proj 52428800 0.001829 signed +xplain1.4-6base-16k-step2319 Layer 38 | self_attn.q_proj Layer 38 self_attn.q_proj 16777216 0.001743 signed +xplain1.4-6base-16k-step2319 Layer 39 | post_attention_layernorm Layer 39 post_attention_layernorm 4096 0.001732 signed +xplain1.4-6base-16k-step2319 Layer 39 | self_attn.q_proj Layer 39 self_attn.q_proj 16777216 0.001715 signed +xplain1.4-6base-16k-step2319 Layer 38 | mlp.down_proj Layer 38 mlp.down_proj 52428800 0.001702 signed +xplain1.4-6base-16k-step2319 Layer 39 | mlp.up_proj Layer 39 mlp.up_proj 52428800 0.001678 signed +xplain1.4-6base-16k-step2319 Layer 36 | input_layernorm Layer 36 input_layernorm 4096 0.001660 signed +xplain1.4-6base-16k-step2319 Layer 37 | mlp.down_proj Layer 37 mlp.down_proj 52428800 0.001632 signed +xplain1.4-6base-16k-step2319 Layer 36 | mlp.gate_proj Layer 36 mlp.gate_proj 52428800 0.001611 signed +xplain1.4-6base-16k-step2319 Layer 37 | post_attention_layernorm Layer 37 post_attention_layernorm 4096 0.001591 signed +xplain1.4-6base-16k-step2319 Layer 37 | self_attn.q_proj Layer 37 self_attn.q_proj 16777216 0.001560 signed +xplain1.4-6base-16k-step2319 Layer 36 | mlp.up_proj Layer 36 mlp.up_proj 52428800 0.001555 signed +xplain1.4-6base-16k-step2319 Layer 35 | mlp.gate_proj Layer 35 mlp.gate_proj 52428800 0.001549 signed +xplain1.4-6base-16k-step2319 Layer 37 | input_layernorm Layer 37 input_layernorm 4096 0.001547 signed +xplain1.4-6base-16k-step2319 Layer 39 | self_attn.k_proj Layer 39 self_attn.k_proj 4194304 0.001546 signed +xplain1.4-6base-16k-step2319 Layer 38 | post_attention_layernorm Layer 38 post_attention_layernorm 4096 0.001541 signed +xplain1.4-6base-16k-step2319 Layer 38 | self_attn.k_proj Layer 38 self_attn.k_proj 4194304 0.001533 signed +xplain1.4-6base-16k-step2319 Layer 00 | input_layernorm Layer 00 input_layernorm 4096 0.001527 signed +xplain1.4-6base-16k-step2319 Layer 34 | mlp.gate_proj Layer 34 mlp.gate_proj 52428800 0.001513 signed +xplain1.4-6base-16k-step2319 Layer 35 | input_layernorm Layer 35 input_layernorm 4096 0.001508 signed +xplain1.4-6base-16k-step2319 Layer 30 | input_layernorm Layer 30 input_layernorm 4096 0.001490 signed +xplain1.4-6base-16k-step2319 Layer 17 | input_layernorm Layer 17 input_layernorm 4096 0.001487 signed +xplain1.4-6base-16k-step2319 Layer 35 | mlp.up_proj Layer 35 mlp.up_proj 52428800 0.001487 signed +xplain1.4-6base-16k-step2319 Layer 34 | mlp.up_proj Layer 34 mlp.up_proj 52428800 0.001458 signed +xplain1.4-6base-16k-step2319 Layer 36 | self_attn.q_proj Layer 36 self_attn.q_proj 16777216 0.001450 signed +xplain1.4-6base-16k-step2319 Layer 33 | mlp.gate_proj Layer 33 mlp.gate_proj 52428800 0.001444 signed +xplain1.4-6base-16k-step2319 Layer 36 | mlp.down_proj Layer 36 mlp.down_proj 52428800 0.001431 signed +xplain1.4-6base-16k-step2319 Layer 34 | self_attn.q_proj Layer 34 self_attn.q_proj 16777216 0.001430 signed +xplain1.4-6base-16k-step2319 Layer 39 | input_layernorm Layer 39 input_layernorm 4096 0.001430 signed +xplain1.4-6base-16k-step2319 Layer 31 | input_layernorm Layer 31 input_layernorm 4096 0.001421 signed +xplain1.4-6base-16k-step2319 Layer 32 | mlp.gate_proj Layer 32 mlp.gate_proj 52428800 0.001418 signed +xplain1.4-6base-16k-step2319 Layer 37 | self_attn.k_proj Layer 37 self_attn.k_proj 4194304 0.001413 signed +xplain1.4-6base-16k-step2319 Layer 31 | mlp.gate_proj Layer 31 mlp.gate_proj 52428800 0.001409 signed +xplain1.4-6base-16k-step2319 Layer 33 | mlp.up_proj Layer 33 mlp.up_proj 52428800 0.001392 signed +xplain1.4-6base-16k-step2319 Layer 34 | mlp.down_proj Layer 34 mlp.down_proj 52428800 0.001389 signed +xplain1.4-6base-16k-step2319 Layer 35 | self_attn.q_proj Layer 35 self_attn.q_proj 16777216 0.001387 signed +xplain1.4-6base-16k-step2319 Layer 30 | mlp.gate_proj Layer 30 mlp.gate_proj 52428800 0.001387 signed +xplain1.4-6base-16k-step2319 Layer 35 | mlp.down_proj Layer 35 mlp.down_proj 52428800 0.001381 signed +xplain1.4-6base-16k-step2319 Layer 29 | mlp.gate_proj Layer 29 mlp.gate_proj 52428800 0.001380 signed +xplain1.4-6base-16k-step2319 Layer 31 | mlp.down_proj Layer 31 mlp.down_proj 52428800 0.001379 signed ### `xplain1.4-6base-32k-step1743` - model sublayer block_label module_type count std stat_mode -xplain1.4-6base-32k-step1743 Final Norm | model.norm.weight Final Norm final_norm 4096 0.006660 signed -xplain1.4-6base-32k-step1743 Layer 22 | input_layernorm Layer 22 input_layernorm 4096 0.005665 signed -xplain1.4-6base-32k-step1743 Layer 28 | input_layernorm Layer 28 input_layernorm 4096 0.005453 signed -xplain1.4-6base-32k-step1743 Layer 25 | input_layernorm Layer 25 input_layernorm 4096 0.005436 signed -xplain1.4-6base-32k-step1743 Layer 18 | input_layernorm Layer 18 input_layernorm 4096 0.004031 signed -xplain1.4-6base-32k-step1743 Layer 21 | input_layernorm Layer 21 input_layernorm 4096 0.003478 signed -xplain1.4-6base-32k-step1743 Layer 38 | input_layernorm Layer 38 input_layernorm 4096 0.002953 signed -xplain1.4-6base-32k-step1743 Layer 38 | mlp.gate_proj Layer 38 mlp.gate_proj 52428800 0.002214 signed -xplain1.4-6base-32k-step1743 Layer 37 | mlp.gate_proj Layer 37 mlp.gate_proj 52428800 0.002141 signed -xplain1.4-6base-32k-step1743 Layer 38 | mlp.up_proj Layer 38 mlp.up_proj 52428800 0.002105 signed -xplain1.4-6base-32k-step1743 Layer 36 | input_layernorm Layer 36 input_layernorm 4096 0.002073 signed -xplain1.4-6base-32k-step1743 Layer 00 | input_layernorm Layer 00 input_layernorm 4096 0.002072 signed -xplain1.4-6base-32k-step1743 Layer 39 | post_attention_layernorm Layer 39 post_attention_layernorm 4096 0.002072 signed -xplain1.4-6base-32k-step1743 Layer 37 | mlp.up_proj Layer 37 mlp.up_proj 52428800 0.002064 signed -xplain1.4-6base-32k-step1743 Layer 39 | mlp.gate_proj Layer 39 mlp.gate_proj 52428800 0.002055 signed -xplain1.4-6base-32k-step1743 Layer 37 | input_layernorm Layer 37 input_layernorm 4096 0.002037 signed -xplain1.4-6base-32k-step1743 Embedding | model.embed_tokens.weight Embedding embed_tokens 201326592 0.002007 signed -xplain1.4-6base-32k-step1743 Layer 38 | self_attn.q_proj Layer 38 self_attn.q_proj 16777216 0.001931 signed -xplain1.4-6base-32k-step1743 Layer 39 | self_attn.q_proj Layer 39 self_attn.q_proj 16777216 0.001905 signed -xplain1.4-6base-32k-step1743 Layer 39 | mlp.up_proj Layer 39 mlp.up_proj 52428800 0.001901 signed -xplain1.4-6base-32k-step1743 Layer 38 | mlp.down_proj Layer 38 mlp.down_proj 52428800 0.001889 signed -xplain1.4-6base-32k-step1743 Layer 36 | mlp.gate_proj Layer 36 mlp.gate_proj 52428800 0.001865 signed -xplain1.4-6base-32k-step1743 Layer 17 | input_layernorm Layer 17 input_layernorm 4096 0.001864 signed -xplain1.4-6base-32k-step1743 Layer 37 | post_attention_layernorm Layer 37 post_attention_layernorm 4096 0.001838 signed -xplain1.4-6base-32k-step1743 Layer 35 | input_layernorm Layer 35 input_layernorm 4096 0.001836 signed -xplain1.4-6base-32k-step1743 Layer 37 | mlp.down_proj Layer 37 mlp.down_proj 52428800 0.001827 signed -xplain1.4-6base-32k-step1743 Layer 30 | input_layernorm Layer 30 input_layernorm 4096 0.001813 signed -xplain1.4-6base-32k-step1743 Layer 36 | mlp.up_proj Layer 36 mlp.up_proj 52428800 0.001812 signed -xplain1.4-6base-32k-step1743 Layer 35 | mlp.gate_proj Layer 35 mlp.gate_proj 52428800 0.001800 signed -xplain1.4-6base-32k-step1743 Layer 34 | mlp.gate_proj Layer 34 mlp.gate_proj 52428800 0.001769 signed -xplain1.4-6base-32k-step1743 Layer 37 | self_attn.q_proj Layer 37 self_attn.q_proj 16777216 0.001763 signed -xplain1.4-6base-32k-step1743 Layer 38 | post_attention_layernorm Layer 38 post_attention_layernorm 4096 0.001744 signed -xplain1.4-6base-32k-step1743 Layer 35 | mlp.up_proj Layer 35 mlp.up_proj 52428800 0.001735 signed -xplain1.4-6base-32k-step1743 Layer 39 | input_layernorm Layer 39 input_layernorm 4096 0.001728 signed -xplain1.4-6base-32k-step1743 Layer 31 | input_layernorm Layer 31 input_layernorm 4096 0.001726 signed -xplain1.4-6base-32k-step1743 Layer 34 | mlp.up_proj Layer 34 mlp.up_proj 52428800 0.001711 signed -xplain1.4-6base-32k-step1743 Layer 39 | self_attn.k_proj Layer 39 self_attn.k_proj 4194304 0.001703 signed -xplain1.4-6base-32k-step1743 Layer 33 | mlp.gate_proj Layer 33 mlp.gate_proj 52428800 0.001699 signed -xplain1.4-6base-32k-step1743 Layer 38 | self_attn.k_proj Layer 38 self_attn.k_proj 4194304 0.001686 signed -xplain1.4-6base-32k-step1743 Layer 19 | input_layernorm Layer 19 input_layernorm 4096 0.001684 signed -xplain1.4-6base-32k-step1743 Layer 36 | self_attn.q_proj Layer 36 self_attn.q_proj 16777216 0.001682 signed -xplain1.4-6base-32k-step1743 Layer 32 | mlp.gate_proj Layer 32 mlp.gate_proj 52428800 0.001675 signed -xplain1.4-6base-32k-step1743 Layer 34 | self_attn.q_proj Layer 34 self_attn.q_proj 16777216 0.001671 signed -xplain1.4-6base-32k-step1743 Layer 31 | mlp.gate_proj Layer 31 mlp.gate_proj 52428800 0.001668 signed -xplain1.4-6base-32k-step1743 Layer 36 | mlp.down_proj Layer 36 mlp.down_proj 52428800 0.001652 signed -xplain1.4-6base-32k-step1743 Layer 30 | mlp.gate_proj Layer 30 mlp.gate_proj 52428800 0.001644 signed -xplain1.4-6base-32k-step1743 Layer 33 | mlp.up_proj Layer 33 mlp.up_proj 52428800 0.001643 signed -xplain1.4-6base-32k-step1743 Layer 32 | input_layernorm Layer 32 input_layernorm 4096 0.001638 signed -xplain1.4-6base-32k-step1743 Layer 29 | mlp.gate_proj Layer 29 mlp.gate_proj 52428800 0.001634 signed -xplain1.4-6base-32k-step1743 Layer 31 | mlp.up_proj Layer 31 mlp.up_proj 52428800 0.001630 signed + model sublayer block_label module_type count std stat_mode +xplain1.4-6base-32k-step1743 Layer 22 | input_layernorm Layer 22 input_layernorm 4096 0.005665 signed +xplain1.4-6base-32k-step1743 Layer 28 | input_layernorm Layer 28 input_layernorm 4096 0.005453 signed +xplain1.4-6base-32k-step1743 Layer 25 | input_layernorm Layer 25 input_layernorm 4096 0.005436 signed +xplain1.4-6base-32k-step1743 Layer 18 | input_layernorm Layer 18 input_layernorm 4096 0.004031 signed +xplain1.4-6base-32k-step1743 Layer 21 | input_layernorm Layer 21 input_layernorm 4096 0.003478 signed +xplain1.4-6base-32k-step1743 Layer 38 | input_layernorm Layer 38 input_layernorm 4096 0.002953 signed +xplain1.4-6base-32k-step1743 Layer 38 | mlp.gate_proj Layer 38 mlp.gate_proj 52428800 0.002214 signed +xplain1.4-6base-32k-step1743 Layer 37 | mlp.gate_proj Layer 37 mlp.gate_proj 52428800 0.002141 signed +xplain1.4-6base-32k-step1743 Layer 38 | mlp.up_proj Layer 38 mlp.up_proj 52428800 0.002105 signed +xplain1.4-6base-32k-step1743 Layer 36 | input_layernorm Layer 36 input_layernorm 4096 0.002073 signed +xplain1.4-6base-32k-step1743 Layer 00 | input_layernorm Layer 00 input_layernorm 4096 0.002072 signed +xplain1.4-6base-32k-step1743 Layer 39 | post_attention_layernorm Layer 39 post_attention_layernorm 4096 0.002072 signed +xplain1.4-6base-32k-step1743 Layer 37 | mlp.up_proj Layer 37 mlp.up_proj 52428800 0.002064 signed +xplain1.4-6base-32k-step1743 Layer 39 | mlp.gate_proj Layer 39 mlp.gate_proj 52428800 0.002055 signed +xplain1.4-6base-32k-step1743 Layer 37 | input_layernorm Layer 37 input_layernorm 4096 0.002037 signed +xplain1.4-6base-32k-step1743 Layer 38 | self_attn.q_proj Layer 38 self_attn.q_proj 16777216 0.001931 signed +xplain1.4-6base-32k-step1743 Layer 39 | self_attn.q_proj Layer 39 self_attn.q_proj 16777216 0.001905 signed +xplain1.4-6base-32k-step1743 Layer 39 | mlp.up_proj Layer 39 mlp.up_proj 52428800 0.001901 signed +xplain1.4-6base-32k-step1743 Layer 38 | mlp.down_proj Layer 38 mlp.down_proj 52428800 0.001889 signed +xplain1.4-6base-32k-step1743 Layer 36 | mlp.gate_proj Layer 36 mlp.gate_proj 52428800 0.001865 signed +xplain1.4-6base-32k-step1743 Layer 17 | input_layernorm Layer 17 input_layernorm 4096 0.001864 signed +xplain1.4-6base-32k-step1743 Layer 37 | post_attention_layernorm Layer 37 post_attention_layernorm 4096 0.001838 signed +xplain1.4-6base-32k-step1743 Layer 35 | input_layernorm Layer 35 input_layernorm 4096 0.001836 signed +xplain1.4-6base-32k-step1743 Layer 37 | mlp.down_proj Layer 37 mlp.down_proj 52428800 0.001827 signed +xplain1.4-6base-32k-step1743 Layer 30 | input_layernorm Layer 30 input_layernorm 4096 0.001813 signed +xplain1.4-6base-32k-step1743 Layer 36 | mlp.up_proj Layer 36 mlp.up_proj 52428800 0.001812 signed +xplain1.4-6base-32k-step1743 Layer 35 | mlp.gate_proj Layer 35 mlp.gate_proj 52428800 0.001800 signed +xplain1.4-6base-32k-step1743 Layer 34 | mlp.gate_proj Layer 34 mlp.gate_proj 52428800 0.001769 signed +xplain1.4-6base-32k-step1743 Layer 37 | self_attn.q_proj Layer 37 self_attn.q_proj 16777216 0.001763 signed +xplain1.4-6base-32k-step1743 Layer 38 | post_attention_layernorm Layer 38 post_attention_layernorm 4096 0.001744 signed +xplain1.4-6base-32k-step1743 Layer 35 | mlp.up_proj Layer 35 mlp.up_proj 52428800 0.001735 signed +xplain1.4-6base-32k-step1743 Layer 39 | input_layernorm Layer 39 input_layernorm 4096 0.001728 signed +xplain1.4-6base-32k-step1743 Layer 31 | input_layernorm Layer 31 input_layernorm 4096 0.001726 signed +xplain1.4-6base-32k-step1743 Layer 34 | mlp.up_proj Layer 34 mlp.up_proj 52428800 0.001711 signed +xplain1.4-6base-32k-step1743 Layer 39 | self_attn.k_proj Layer 39 self_attn.k_proj 4194304 0.001703 signed +xplain1.4-6base-32k-step1743 Layer 33 | mlp.gate_proj Layer 33 mlp.gate_proj 52428800 0.001699 signed +xplain1.4-6base-32k-step1743 Layer 38 | self_attn.k_proj Layer 38 self_attn.k_proj 4194304 0.001686 signed +xplain1.4-6base-32k-step1743 Layer 19 | input_layernorm Layer 19 input_layernorm 4096 0.001684 signed +xplain1.4-6base-32k-step1743 Layer 36 | self_attn.q_proj Layer 36 self_attn.q_proj 16777216 0.001682 signed +xplain1.4-6base-32k-step1743 Layer 32 | mlp.gate_proj Layer 32 mlp.gate_proj 52428800 0.001675 signed +xplain1.4-6base-32k-step1743 Layer 34 | self_attn.q_proj Layer 34 self_attn.q_proj 16777216 0.001671 signed +xplain1.4-6base-32k-step1743 Layer 31 | mlp.gate_proj Layer 31 mlp.gate_proj 52428800 0.001668 signed +xplain1.4-6base-32k-step1743 Layer 36 | mlp.down_proj Layer 36 mlp.down_proj 52428800 0.001652 signed +xplain1.4-6base-32k-step1743 Layer 30 | mlp.gate_proj Layer 30 mlp.gate_proj 52428800 0.001644 signed +xplain1.4-6base-32k-step1743 Layer 33 | mlp.up_proj Layer 33 mlp.up_proj 52428800 0.001643 signed +xplain1.4-6base-32k-step1743 Layer 32 | input_layernorm Layer 32 input_layernorm 4096 0.001638 signed +xplain1.4-6base-32k-step1743 Layer 29 | mlp.gate_proj Layer 29 mlp.gate_proj 52428800 0.001634 signed +xplain1.4-6base-32k-step1743 Layer 31 | mlp.up_proj Layer 31 mlp.up_proj 52428800 0.001630 signed +xplain1.4-6base-32k-step1743 Layer 35 | self_attn.q_proj Layer 35 self_attn.q_proj 16777216 0.001630 signed +xplain1.4-6base-32k-step1743 Layer 29 | mlp.down_proj Layer 29 mlp.down_proj 52428800 0.001628 signed ### `xplain1.4-6base-64k-step1743` - model sublayer block_label module_type count std stat_mode -xplain1.4-6base-64k-step1743 Final Norm | model.norm.weight Final Norm final_norm 4096 0.007889 signed -xplain1.4-6base-64k-step1743 Layer 22 | input_layernorm Layer 22 input_layernorm 4096 0.007021 signed -xplain1.4-6base-64k-step1743 Layer 28 | input_layernorm Layer 28 input_layernorm 4096 0.006637 signed -xplain1.4-6base-64k-step1743 Layer 25 | input_layernorm Layer 25 input_layernorm 4096 0.006622 signed -xplain1.4-6base-64k-step1743 Layer 18 | input_layernorm Layer 18 input_layernorm 4096 0.004975 signed -xplain1.4-6base-64k-step1743 Layer 21 | input_layernorm Layer 21 input_layernorm 4096 0.004198 signed -xplain1.4-6base-64k-step1743 Layer 38 | input_layernorm Layer 38 input_layernorm 4096 0.003582 signed -xplain1.4-6base-64k-step1743 Layer 37 | input_layernorm Layer 37 input_layernorm 4096 0.002491 signed -xplain1.4-6base-64k-step1743 Layer 36 | input_layernorm Layer 36 input_layernorm 4096 0.002454 signed -xplain1.4-6base-64k-step1743 Layer 00 | input_layernorm Layer 00 input_layernorm 4096 0.002429 signed -xplain1.4-6base-64k-step1743 Layer 38 | mlp.gate_proj Layer 38 mlp.gate_proj 52428800 0.002357 signed -xplain1.4-6base-64k-step1743 Layer 39 | post_attention_layernorm Layer 39 post_attention_layernorm 4096 0.002316 signed -xplain1.4-6base-64k-step1743 Layer 37 | mlp.gate_proj Layer 37 mlp.gate_proj 52428800 0.002259 signed -xplain1.4-6base-64k-step1743 Layer 38 | mlp.up_proj Layer 38 mlp.up_proj 52428800 0.002255 signed -xplain1.4-6base-64k-step1743 Embedding | model.embed_tokens.weight Embedding embed_tokens 201326592 0.002247 signed -xplain1.4-6base-64k-step1743 Layer 39 | mlp.gate_proj Layer 39 mlp.gate_proj 52428800 0.002206 signed -xplain1.4-6base-64k-step1743 Layer 37 | mlp.up_proj Layer 37 mlp.up_proj 52428800 0.002186 signed -xplain1.4-6base-64k-step1743 Layer 17 | input_layernorm Layer 17 input_layernorm 4096 0.002164 signed -xplain1.4-6base-64k-step1743 Layer 35 | input_layernorm Layer 35 input_layernorm 4096 0.002126 signed -xplain1.4-6base-64k-step1743 Layer 39 | mlp.up_proj Layer 39 mlp.up_proj 52428800 0.002050 signed -xplain1.4-6base-64k-step1743 Layer 30 | input_layernorm Layer 30 input_layernorm 4096 0.002045 signed -xplain1.4-6base-64k-step1743 Layer 39 | self_attn.q_proj Layer 39 self_attn.q_proj 16777216 0.002030 signed -xplain1.4-6base-64k-step1743 Layer 38 | self_attn.q_proj Layer 38 self_attn.q_proj 16777216 0.002021 signed -xplain1.4-6base-64k-step1743 Layer 38 | mlp.down_proj Layer 38 mlp.down_proj 52428800 0.002011 signed -xplain1.4-6base-64k-step1743 Layer 36 | mlp.gate_proj Layer 36 mlp.gate_proj 52428800 0.001999 signed -xplain1.4-6base-64k-step1743 Layer 19 | input_layernorm Layer 19 input_layernorm 4096 0.001980 signed -xplain1.4-6base-64k-step1743 Layer 39 | input_layernorm Layer 39 input_layernorm 4096 0.001972 signed -xplain1.4-6base-64k-step1743 Layer 31 | input_layernorm Layer 31 input_layernorm 4096 0.001948 signed -xplain1.4-6base-64k-step1743 Layer 36 | mlp.up_proj Layer 36 mlp.up_proj 52428800 0.001947 signed -xplain1.4-6base-64k-step1743 Layer 37 | post_attention_layernorm Layer 37 post_attention_layernorm 4096 0.001946 signed -xplain1.4-6base-64k-step1743 Layer 35 | mlp.gate_proj Layer 35 mlp.gate_proj 52428800 0.001936 signed -xplain1.4-6base-64k-step1743 Layer 37 | mlp.down_proj Layer 37 mlp.down_proj 52428800 0.001930 signed -xplain1.4-6base-64k-step1743 Layer 34 | mlp.gate_proj Layer 34 mlp.gate_proj 52428800 0.001911 signed -xplain1.4-6base-64k-step1743 Layer 33 | input_layernorm Layer 33 input_layernorm 4096 0.001875 signed -xplain1.4-6base-64k-step1743 Layer 38 | post_attention_layernorm Layer 38 post_attention_layernorm 4096 0.001872 signed -xplain1.4-6base-64k-step1743 Layer 37 | self_attn.q_proj Layer 37 self_attn.q_proj 16777216 0.001868 signed -xplain1.4-6base-64k-step1743 Layer 35 | mlp.up_proj Layer 35 mlp.up_proj 52428800 0.001866 signed -xplain1.4-6base-64k-step1743 Layer 14 | input_layernorm Layer 14 input_layernorm 4096 0.001866 signed -xplain1.4-6base-64k-step1743 Layer 32 | input_layernorm Layer 32 input_layernorm 4096 0.001862 signed -xplain1.4-6base-64k-step1743 Layer 34 | mlp.up_proj Layer 34 mlp.up_proj 52428800 0.001846 signed -xplain1.4-6base-64k-step1743 Layer 33 | mlp.gate_proj Layer 33 mlp.gate_proj 52428800 0.001843 signed -xplain1.4-6base-64k-step1743 Layer 15 | input_layernorm Layer 15 input_layernorm 4096 0.001836 signed -xplain1.4-6base-64k-step1743 Layer 32 | mlp.gate_proj Layer 32 mlp.gate_proj 52428800 0.001823 signed -xplain1.4-6base-64k-step1743 Layer 31 | mlp.gate_proj Layer 31 mlp.gate_proj 52428800 0.001822 signed -xplain1.4-6base-64k-step1743 Layer 34 | self_attn.q_proj Layer 34 self_attn.q_proj 16777216 0.001816 signed -xplain1.4-6base-64k-step1743 Layer 36 | self_attn.q_proj Layer 36 self_attn.q_proj 16777216 0.001804 signed -xplain1.4-6base-64k-step1743 Layer 30 | mlp.gate_proj Layer 30 mlp.gate_proj 52428800 0.001799 signed -xplain1.4-6base-64k-step1743 Layer 39 | self_attn.k_proj Layer 39 self_attn.k_proj 4194304 0.001799 signed -xplain1.4-6base-64k-step1743 Layer 29 | mlp.gate_proj Layer 29 mlp.gate_proj 52428800 0.001790 signed -xplain1.4-6base-64k-step1743 Layer 28 | mlp.gate_proj Layer 28 mlp.gate_proj 52428800 0.001780 signed + model sublayer block_label module_type count std stat_mode +xplain1.4-6base-64k-step1743 Layer 22 | input_layernorm Layer 22 input_layernorm 4096 0.007021 signed +xplain1.4-6base-64k-step1743 Layer 28 | input_layernorm Layer 28 input_layernorm 4096 0.006637 signed +xplain1.4-6base-64k-step1743 Layer 25 | input_layernorm Layer 25 input_layernorm 4096 0.006622 signed +xplain1.4-6base-64k-step1743 Layer 18 | input_layernorm Layer 18 input_layernorm 4096 0.004975 signed +xplain1.4-6base-64k-step1743 Layer 21 | input_layernorm Layer 21 input_layernorm 4096 0.004198 signed +xplain1.4-6base-64k-step1743 Layer 38 | input_layernorm Layer 38 input_layernorm 4096 0.003582 signed +xplain1.4-6base-64k-step1743 Layer 37 | input_layernorm Layer 37 input_layernorm 4096 0.002491 signed +xplain1.4-6base-64k-step1743 Layer 36 | input_layernorm Layer 36 input_layernorm 4096 0.002454 signed +xplain1.4-6base-64k-step1743 Layer 00 | input_layernorm Layer 00 input_layernorm 4096 0.002429 signed +xplain1.4-6base-64k-step1743 Layer 38 | mlp.gate_proj Layer 38 mlp.gate_proj 52428800 0.002357 signed +xplain1.4-6base-64k-step1743 Layer 39 | post_attention_layernorm Layer 39 post_attention_layernorm 4096 0.002316 signed +xplain1.4-6base-64k-step1743 Layer 37 | mlp.gate_proj Layer 37 mlp.gate_proj 52428800 0.002259 signed +xplain1.4-6base-64k-step1743 Layer 38 | mlp.up_proj Layer 38 mlp.up_proj 52428800 0.002255 signed +xplain1.4-6base-64k-step1743 Layer 39 | mlp.gate_proj Layer 39 mlp.gate_proj 52428800 0.002206 signed +xplain1.4-6base-64k-step1743 Layer 37 | mlp.up_proj Layer 37 mlp.up_proj 52428800 0.002186 signed +xplain1.4-6base-64k-step1743 Layer 17 | input_layernorm Layer 17 input_layernorm 4096 0.002164 signed +xplain1.4-6base-64k-step1743 Layer 35 | input_layernorm Layer 35 input_layernorm 4096 0.002126 signed +xplain1.4-6base-64k-step1743 Layer 39 | mlp.up_proj Layer 39 mlp.up_proj 52428800 0.002050 signed +xplain1.4-6base-64k-step1743 Layer 30 | input_layernorm Layer 30 input_layernorm 4096 0.002045 signed +xplain1.4-6base-64k-step1743 Layer 39 | self_attn.q_proj Layer 39 self_attn.q_proj 16777216 0.002030 signed +xplain1.4-6base-64k-step1743 Layer 38 | self_attn.q_proj Layer 38 self_attn.q_proj 16777216 0.002021 signed +xplain1.4-6base-64k-step1743 Layer 38 | mlp.down_proj Layer 38 mlp.down_proj 52428800 0.002011 signed +xplain1.4-6base-64k-step1743 Layer 36 | mlp.gate_proj Layer 36 mlp.gate_proj 52428800 0.001999 signed +xplain1.4-6base-64k-step1743 Layer 19 | input_layernorm Layer 19 input_layernorm 4096 0.001980 signed +xplain1.4-6base-64k-step1743 Layer 39 | input_layernorm Layer 39 input_layernorm 4096 0.001972 signed +xplain1.4-6base-64k-step1743 Layer 31 | input_layernorm Layer 31 input_layernorm 4096 0.001948 signed +xplain1.4-6base-64k-step1743 Layer 36 | mlp.up_proj Layer 36 mlp.up_proj 52428800 0.001947 signed +xplain1.4-6base-64k-step1743 Layer 37 | post_attention_layernorm Layer 37 post_attention_layernorm 4096 0.001946 signed +xplain1.4-6base-64k-step1743 Layer 35 | mlp.gate_proj Layer 35 mlp.gate_proj 52428800 0.001936 signed +xplain1.4-6base-64k-step1743 Layer 37 | mlp.down_proj Layer 37 mlp.down_proj 52428800 0.001930 signed +xplain1.4-6base-64k-step1743 Layer 34 | mlp.gate_proj Layer 34 mlp.gate_proj 52428800 0.001911 signed +xplain1.4-6base-64k-step1743 Layer 33 | input_layernorm Layer 33 input_layernorm 4096 0.001875 signed +xplain1.4-6base-64k-step1743 Layer 38 | post_attention_layernorm Layer 38 post_attention_layernorm 4096 0.001872 signed +xplain1.4-6base-64k-step1743 Layer 37 | self_attn.q_proj Layer 37 self_attn.q_proj 16777216 0.001868 signed +xplain1.4-6base-64k-step1743 Layer 35 | mlp.up_proj Layer 35 mlp.up_proj 52428800 0.001866 signed +xplain1.4-6base-64k-step1743 Layer 14 | input_layernorm Layer 14 input_layernorm 4096 0.001866 signed +xplain1.4-6base-64k-step1743 Layer 32 | input_layernorm Layer 32 input_layernorm 4096 0.001862 signed +xplain1.4-6base-64k-step1743 Layer 34 | mlp.up_proj Layer 34 mlp.up_proj 52428800 0.001846 signed +xplain1.4-6base-64k-step1743 Layer 33 | mlp.gate_proj Layer 33 mlp.gate_proj 52428800 0.001843 signed +xplain1.4-6base-64k-step1743 Layer 15 | input_layernorm Layer 15 input_layernorm 4096 0.001836 signed +xplain1.4-6base-64k-step1743 Layer 32 | mlp.gate_proj Layer 32 mlp.gate_proj 52428800 0.001823 signed +xplain1.4-6base-64k-step1743 Layer 31 | mlp.gate_proj Layer 31 mlp.gate_proj 52428800 0.001822 signed +xplain1.4-6base-64k-step1743 Layer 34 | self_attn.q_proj Layer 34 self_attn.q_proj 16777216 0.001816 signed +xplain1.4-6base-64k-step1743 Layer 36 | self_attn.q_proj Layer 36 self_attn.q_proj 16777216 0.001804 signed +xplain1.4-6base-64k-step1743 Layer 30 | mlp.gate_proj Layer 30 mlp.gate_proj 52428800 0.001799 signed +xplain1.4-6base-64k-step1743 Layer 39 | self_attn.k_proj Layer 39 self_attn.k_proj 4194304 0.001799 signed +xplain1.4-6base-64k-step1743 Layer 29 | mlp.gate_proj Layer 29 mlp.gate_proj 52428800 0.001790 signed +xplain1.4-6base-64k-step1743 Layer 28 | mlp.gate_proj Layer 28 mlp.gate_proj 52428800 0.001780 signed +xplain1.4-6base-64k-step1743 Layer 33 | mlp.up_proj Layer 33 mlp.up_proj 52428800 0.001778 signed +xplain1.4-6base-64k-step1743 Layer 31 | mlp.up_proj Layer 31 mlp.up_proj 52428800 0.001776 signed ### `xplain1.4-6base-8k-step2319` - model sublayer block_label module_type count std stat_mode -xplain1.4-6base-8k-step2319 Final Norm | model.norm.weight Final Norm final_norm 4096 0.003269 signed -xplain1.4-6base-8k-step2319 Layer 28 | input_layernorm Layer 28 input_layernorm 4096 0.002199 signed -xplain1.4-6base-8k-step2319 Layer 22 | input_layernorm Layer 22 input_layernorm 4096 0.002092 signed -xplain1.4-6base-8k-step2319 Layer 25 | input_layernorm Layer 25 input_layernorm 4096 0.002085 signed -xplain1.4-6base-8k-step2319 Layer 18 | input_layernorm Layer 18 input_layernorm 4096 0.001681 signed -xplain1.4-6base-8k-step2319 Layer 38 | mlp.gate_proj Layer 38 mlp.gate_proj 52428800 0.001586 signed -xplain1.4-6base-8k-step2319 Layer 37 | mlp.gate_proj Layer 37 mlp.gate_proj 52428800 0.001515 signed -xplain1.4-6base-8k-step2319 Layer 38 | mlp.up_proj Layer 38 mlp.up_proj 52428800 0.001469 signed -xplain1.4-6base-8k-step2319 Layer 37 | mlp.up_proj Layer 37 mlp.up_proj 52428800 0.001428 signed -xplain1.4-6base-8k-step2319 Layer 39 | mlp.gate_proj Layer 39 mlp.gate_proj 52428800 0.001405 signed -xplain1.4-6base-8k-step2319 Layer 38 | self_attn.q_proj Layer 38 self_attn.q_proj 16777216 0.001402 signed -xplain1.4-6base-8k-step2319 Layer 21 | input_layernorm Layer 21 input_layernorm 4096 0.001373 signed -xplain1.4-6base-8k-step2319 Layer 39 | self_attn.q_proj Layer 39 self_attn.q_proj 16777216 0.001353 signed -xplain1.4-6base-8k-step2319 Layer 38 | mlp.down_proj Layer 38 mlp.down_proj 52428800 0.001327 signed -xplain1.4-6base-8k-step2319 Layer 37 | mlp.down_proj Layer 37 mlp.down_proj 52428800 0.001265 signed -xplain1.4-6base-8k-step2319 Layer 39 | mlp.up_proj Layer 39 mlp.up_proj 52428800 0.001259 signed -xplain1.4-6base-8k-step2319 Layer 38 | self_attn.k_proj Layer 38 self_attn.k_proj 4194304 0.001250 signed -xplain1.4-6base-8k-step2319 Layer 39 | self_attn.k_proj Layer 39 self_attn.k_proj 4194304 0.001244 signed -xplain1.4-6base-8k-step2319 Layer 38 | input_layernorm Layer 38 input_layernorm 4096 0.001241 signed -xplain1.4-6base-8k-step2319 Layer 36 | mlp.gate_proj Layer 36 mlp.gate_proj 52428800 0.001226 signed -xplain1.4-6base-8k-step2319 Layer 37 | self_attn.q_proj Layer 37 self_attn.q_proj 16777216 0.001212 signed -xplain1.4-6base-8k-step2319 Layer 38 | post_attention_layernorm Layer 38 post_attention_layernorm 4096 0.001187 signed -xplain1.4-6base-8k-step2319 Embedding | model.embed_tokens.weight Embedding embed_tokens 201326592 0.001185 signed -xplain1.4-6base-8k-step2319 Layer 37 | post_attention_layernorm Layer 37 post_attention_layernorm 4096 0.001183 signed -xplain1.4-6base-8k-step2319 Layer 35 | mlp.gate_proj Layer 35 mlp.gate_proj 52428800 0.001175 signed -xplain1.4-6base-8k-step2319 Layer 36 | mlp.up_proj Layer 36 mlp.up_proj 52428800 0.001160 signed -xplain1.4-6base-8k-step2319 Layer 34 | mlp.gate_proj Layer 34 mlp.gate_proj 52428800 0.001139 signed -xplain1.4-6base-8k-step2319 Layer 23 | input_layernorm Layer 23 input_layernorm 4096 0.001125 signed -xplain1.4-6base-8k-step2319 Layer 35 | mlp.up_proj Layer 35 mlp.up_proj 52428800 0.001116 signed -xplain1.4-6base-8k-step2319 Layer 37 | self_attn.k_proj Layer 37 self_attn.k_proj 4194304 0.001100 signed -xplain1.4-6base-8k-step2319 Layer 34 | mlp.up_proj Layer 34 mlp.up_proj 52428800 0.001088 signed -xplain1.4-6base-8k-step2319 Layer 36 | self_attn.q_proj Layer 36 self_attn.q_proj 16777216 0.001080 signed -xplain1.4-6base-8k-step2319 Layer 33 | mlp.gate_proj Layer 33 mlp.gate_proj 52428800 0.001072 signed -xplain1.4-6base-8k-step2319 Layer 34 | self_attn.q_proj Layer 34 self_attn.q_proj 16777216 0.001070 signed -xplain1.4-6base-8k-step2319 Layer 26 | input_layernorm Layer 26 input_layernorm 4096 0.001063 signed -xplain1.4-6base-8k-step2319 Layer 36 | mlp.down_proj Layer 36 mlp.down_proj 52428800 0.001055 signed -xplain1.4-6base-8k-step2319 Layer 32 | mlp.gate_proj Layer 32 mlp.gate_proj 52428800 0.001048 signed -xplain1.4-6base-8k-step2319 Layer 31 | mlp.gate_proj Layer 31 mlp.gate_proj 52428800 0.001041 signed -xplain1.4-6base-8k-step2319 Layer 18 | mlp.gate_proj Layer 18 mlp.gate_proj 52428800 0.001035 signed -xplain1.4-6base-8k-step2319 Layer 19 | mlp.gate_proj Layer 19 mlp.gate_proj 52428800 0.001032 signed -xplain1.4-6base-8k-step2319 Layer 33 | mlp.up_proj Layer 33 mlp.up_proj 52428800 0.001030 signed -xplain1.4-6base-8k-step2319 Layer 34 | mlp.down_proj Layer 34 mlp.down_proj 52428800 0.001028 signed -xplain1.4-6base-8k-step2319 Layer 17 | mlp.gate_proj Layer 17 mlp.gate_proj 52428800 0.001027 signed -xplain1.4-6base-8k-step2319 Layer 39 | post_attention_layernorm Layer 39 post_attention_layernorm 4096 0.001026 signed -xplain1.4-6base-8k-step2319 Layer 30 | mlp.gate_proj Layer 30 mlp.gate_proj 52428800 0.001026 signed -xplain1.4-6base-8k-step2319 Layer 35 | mlp.down_proj Layer 35 mlp.down_proj 52428800 0.001025 signed -xplain1.4-6base-8k-step2319 Layer 29 | mlp.gate_proj Layer 29 mlp.gate_proj 52428800 0.001024 signed -xplain1.4-6base-8k-step2319 Layer 28 | mlp.gate_proj Layer 28 mlp.gate_proj 52428800 0.001023 signed -xplain1.4-6base-8k-step2319 Layer 27 | mlp.gate_proj Layer 27 mlp.gate_proj 52428800 0.001022 signed -xplain1.4-6base-8k-step2319 Layer 35 | self_attn.q_proj Layer 35 self_attn.q_proj 16777216 0.001021 signed + model sublayer block_label module_type count std stat_mode +xplain1.4-6base-8k-step2319 Layer 28 | input_layernorm Layer 28 input_layernorm 4096 0.002199 signed +xplain1.4-6base-8k-step2319 Layer 22 | input_layernorm Layer 22 input_layernorm 4096 0.002092 signed +xplain1.4-6base-8k-step2319 Layer 25 | input_layernorm Layer 25 input_layernorm 4096 0.002085 signed +xplain1.4-6base-8k-step2319 Layer 18 | input_layernorm Layer 18 input_layernorm 4096 0.001681 signed +xplain1.4-6base-8k-step2319 Layer 38 | mlp.gate_proj Layer 38 mlp.gate_proj 52428800 0.001586 signed +xplain1.4-6base-8k-step2319 Layer 37 | mlp.gate_proj Layer 37 mlp.gate_proj 52428800 0.001515 signed +xplain1.4-6base-8k-step2319 Layer 38 | mlp.up_proj Layer 38 mlp.up_proj 52428800 0.001469 signed +xplain1.4-6base-8k-step2319 Layer 37 | mlp.up_proj Layer 37 mlp.up_proj 52428800 0.001428 signed +xplain1.4-6base-8k-step2319 Layer 39 | mlp.gate_proj Layer 39 mlp.gate_proj 52428800 0.001405 signed +xplain1.4-6base-8k-step2319 Layer 38 | self_attn.q_proj Layer 38 self_attn.q_proj 16777216 0.001402 signed +xplain1.4-6base-8k-step2319 Layer 21 | input_layernorm Layer 21 input_layernorm 4096 0.001373 signed +xplain1.4-6base-8k-step2319 Layer 39 | self_attn.q_proj Layer 39 self_attn.q_proj 16777216 0.001353 signed +xplain1.4-6base-8k-step2319 Layer 38 | mlp.down_proj Layer 38 mlp.down_proj 52428800 0.001327 signed +xplain1.4-6base-8k-step2319 Layer 37 | mlp.down_proj Layer 37 mlp.down_proj 52428800 0.001265 signed +xplain1.4-6base-8k-step2319 Layer 39 | mlp.up_proj Layer 39 mlp.up_proj 52428800 0.001259 signed +xplain1.4-6base-8k-step2319 Layer 38 | self_attn.k_proj Layer 38 self_attn.k_proj 4194304 0.001250 signed +xplain1.4-6base-8k-step2319 Layer 39 | self_attn.k_proj Layer 39 self_attn.k_proj 4194304 0.001244 signed +xplain1.4-6base-8k-step2319 Layer 38 | input_layernorm Layer 38 input_layernorm 4096 0.001241 signed +xplain1.4-6base-8k-step2319 Layer 36 | mlp.gate_proj Layer 36 mlp.gate_proj 52428800 0.001226 signed +xplain1.4-6base-8k-step2319 Layer 37 | self_attn.q_proj Layer 37 self_attn.q_proj 16777216 0.001212 signed +xplain1.4-6base-8k-step2319 Layer 38 | post_attention_layernorm Layer 38 post_attention_layernorm 4096 0.001187 signed +xplain1.4-6base-8k-step2319 Layer 37 | post_attention_layernorm Layer 37 post_attention_layernorm 4096 0.001183 signed +xplain1.4-6base-8k-step2319 Layer 35 | mlp.gate_proj Layer 35 mlp.gate_proj 52428800 0.001175 signed +xplain1.4-6base-8k-step2319 Layer 36 | mlp.up_proj Layer 36 mlp.up_proj 52428800 0.001160 signed +xplain1.4-6base-8k-step2319 Layer 34 | mlp.gate_proj Layer 34 mlp.gate_proj 52428800 0.001139 signed +xplain1.4-6base-8k-step2319 Layer 23 | input_layernorm Layer 23 input_layernorm 4096 0.001125 signed +xplain1.4-6base-8k-step2319 Layer 35 | mlp.up_proj Layer 35 mlp.up_proj 52428800 0.001116 signed +xplain1.4-6base-8k-step2319 Layer 37 | self_attn.k_proj Layer 37 self_attn.k_proj 4194304 0.001100 signed +xplain1.4-6base-8k-step2319 Layer 34 | mlp.up_proj Layer 34 mlp.up_proj 52428800 0.001088 signed +xplain1.4-6base-8k-step2319 Layer 36 | self_attn.q_proj Layer 36 self_attn.q_proj 16777216 0.001080 signed +xplain1.4-6base-8k-step2319 Layer 33 | mlp.gate_proj Layer 33 mlp.gate_proj 52428800 0.001072 signed +xplain1.4-6base-8k-step2319 Layer 34 | self_attn.q_proj Layer 34 self_attn.q_proj 16777216 0.001070 signed +xplain1.4-6base-8k-step2319 Layer 26 | input_layernorm Layer 26 input_layernorm 4096 0.001063 signed +xplain1.4-6base-8k-step2319 Layer 36 | mlp.down_proj Layer 36 mlp.down_proj 52428800 0.001055 signed +xplain1.4-6base-8k-step2319 Layer 32 | mlp.gate_proj Layer 32 mlp.gate_proj 52428800 0.001048 signed +xplain1.4-6base-8k-step2319 Layer 31 | mlp.gate_proj Layer 31 mlp.gate_proj 52428800 0.001041 signed +xplain1.4-6base-8k-step2319 Layer 18 | mlp.gate_proj Layer 18 mlp.gate_proj 52428800 0.001035 signed +xplain1.4-6base-8k-step2319 Layer 19 | mlp.gate_proj Layer 19 mlp.gate_proj 52428800 0.001032 signed +xplain1.4-6base-8k-step2319 Layer 33 | mlp.up_proj Layer 33 mlp.up_proj 52428800 0.001030 signed +xplain1.4-6base-8k-step2319 Layer 34 | mlp.down_proj Layer 34 mlp.down_proj 52428800 0.001028 signed +xplain1.4-6base-8k-step2319 Layer 17 | mlp.gate_proj Layer 17 mlp.gate_proj 52428800 0.001027 signed +xplain1.4-6base-8k-step2319 Layer 39 | post_attention_layernorm Layer 39 post_attention_layernorm 4096 0.001026 signed +xplain1.4-6base-8k-step2319 Layer 30 | mlp.gate_proj Layer 30 mlp.gate_proj 52428800 0.001026 signed +xplain1.4-6base-8k-step2319 Layer 35 | mlp.down_proj Layer 35 mlp.down_proj 52428800 0.001025 signed +xplain1.4-6base-8k-step2319 Layer 29 | mlp.gate_proj Layer 29 mlp.gate_proj 52428800 0.001024 signed +xplain1.4-6base-8k-step2319 Layer 28 | mlp.gate_proj Layer 28 mlp.gate_proj 52428800 0.001023 signed +xplain1.4-6base-8k-step2319 Layer 27 | mlp.gate_proj Layer 27 mlp.gate_proj 52428800 0.001022 signed +xplain1.4-6base-8k-step2319 Layer 35 | self_attn.q_proj Layer 35 self_attn.q_proj 16777216 0.001021 signed +xplain1.4-6base-8k-step2319 Layer 25 | mlp.gate_proj Layer 25 mlp.gate_proj 52428800 0.001020 signed +xplain1.4-6base-8k-step2319 Layer 26 | mlp.gate_proj Layer 26 mlp.gate_proj 52428800 0.001019 signed ### `xplain1.4-7base-32k-step1743` - model sublayer block_label module_type count std stat_mode -xplain1.4-7base-32k-step1743 Final Norm | model.norm.weight Final Norm final_norm 4096 0.006105 signed -xplain1.4-7base-32k-step1743 Layer 22 | input_layernorm Layer 22 input_layernorm 4096 0.004702 signed -xplain1.4-7base-32k-step1743 Layer 28 | input_layernorm Layer 28 input_layernorm 4096 0.004587 signed -xplain1.4-7base-32k-step1743 Layer 25 | input_layernorm Layer 25 input_layernorm 4096 0.004584 signed -xplain1.4-7base-32k-step1743 Layer 18 | input_layernorm Layer 18 input_layernorm 4096 0.004540 signed -xplain1.4-7base-32k-step1743 Layer 21 | input_layernorm Layer 21 input_layernorm 4096 0.002993 signed -xplain1.4-7base-32k-step1743 Layer 38 | input_layernorm Layer 38 input_layernorm 4096 0.002686 signed -xplain1.4-7base-32k-step1743 Layer 17 | input_layernorm Layer 17 input_layernorm 4096 0.002019 signed -xplain1.4-7base-32k-step1743 Layer 36 | input_layernorm Layer 36 input_layernorm 4096 0.001978 signed -xplain1.4-7base-32k-step1743 Layer 37 | input_layernorm Layer 37 input_layernorm 4096 0.001967 signed -xplain1.4-7base-32k-step1743 Layer 16 | input_layernorm Layer 16 input_layernorm 4096 0.001832 signed -xplain1.4-7base-32k-step1743 Layer 14 | input_layernorm Layer 14 input_layernorm 4096 0.001804 signed -xplain1.4-7base-32k-step1743 Layer 15 | input_layernorm Layer 15 input_layernorm 4096 0.001766 signed -xplain1.4-7base-32k-step1743 Layer 38 | mlp.gate_proj Layer 38 mlp.gate_proj 52428800 0.001752 signed -xplain1.4-7base-32k-step1743 Layer 37 | mlp.gate_proj Layer 37 mlp.gate_proj 52428800 0.001746 signed -xplain1.4-7base-32k-step1743 Layer 36 | mlp.gate_proj Layer 36 mlp.gate_proj 52428800 0.001724 signed -xplain1.4-7base-32k-step1743 Embedding | model.embed_tokens.weight Embedding embed_tokens 201326592 0.001716 signed -xplain1.4-7base-32k-step1743 Layer 37 | mlp.up_proj Layer 37 mlp.up_proj 52428800 0.001685 signed -xplain1.4-7base-32k-step1743 Layer 38 | mlp.up_proj Layer 38 mlp.up_proj 52428800 0.001680 signed -xplain1.4-7base-32k-step1743 Layer 36 | mlp.up_proj Layer 36 mlp.up_proj 52428800 0.001668 signed -xplain1.4-7base-32k-step1743 Layer 39 | post_attention_layernorm Layer 39 post_attention_layernorm 4096 0.001665 signed -xplain1.4-7base-32k-step1743 Layer 39 | mlp.gate_proj Layer 39 mlp.gate_proj 52428800 0.001665 signed -xplain1.4-7base-32k-step1743 Layer 35 | mlp.gate_proj Layer 35 mlp.gate_proj 52428800 0.001662 signed -xplain1.4-7base-32k-step1743 Layer 34 | mlp.gate_proj Layer 34 mlp.gate_proj 52428800 0.001657 signed -xplain1.4-7base-32k-step1743 Layer 19 | input_layernorm Layer 19 input_layernorm 4096 0.001644 signed -xplain1.4-7base-32k-step1743 Layer 13 | input_layernorm Layer 13 input_layernorm 4096 0.001611 signed -xplain1.4-7base-32k-step1743 Layer 00 | input_layernorm Layer 00 input_layernorm 4096 0.001606 signed -xplain1.4-7base-32k-step1743 Layer 35 | mlp.up_proj Layer 35 mlp.up_proj 52428800 0.001599 signed -xplain1.4-7base-32k-step1743 Layer 34 | mlp.up_proj Layer 34 mlp.up_proj 52428800 0.001599 signed -xplain1.4-7base-32k-step1743 Layer 33 | mlp.gate_proj Layer 33 mlp.gate_proj 52428800 0.001592 signed -xplain1.4-7base-32k-step1743 Layer 34 | self_attn.q_proj Layer 34 self_attn.q_proj 16777216 0.001577 signed -xplain1.4-7base-32k-step1743 Layer 36 | self_attn.q_proj Layer 36 self_attn.q_proj 16777216 0.001575 signed -xplain1.4-7base-32k-step1743 Layer 32 | mlp.gate_proj Layer 32 mlp.gate_proj 52428800 0.001565 signed -xplain1.4-7base-32k-step1743 Layer 31 | mlp.gate_proj Layer 31 mlp.gate_proj 52428800 0.001558 signed -xplain1.4-7base-32k-step1743 Layer 35 | self_attn.q_proj Layer 35 self_attn.q_proj 16777216 0.001557 signed -xplain1.4-7base-32k-step1743 Layer 38 | self_attn.q_proj Layer 38 self_attn.q_proj 16777216 0.001556 signed -xplain1.4-7base-32k-step1743 Layer 39 | mlp.up_proj Layer 39 mlp.up_proj 52428800 0.001542 signed -xplain1.4-7base-32k-step1743 Layer 30 | input_layernorm Layer 30 input_layernorm 4096 0.001540 signed -xplain1.4-7base-32k-step1743 Layer 30 | mlp.gate_proj Layer 30 mlp.gate_proj 52428800 0.001533 signed -xplain1.4-7base-32k-step1743 Layer 33 | mlp.up_proj Layer 33 mlp.up_proj 52428800 0.001533 signed -xplain1.4-7base-32k-step1743 Layer 31 | mlp.down_proj Layer 31 mlp.down_proj 52428800 0.001528 signed -xplain1.4-7base-32k-step1743 Layer 30 | mlp.down_proj Layer 30 mlp.down_proj 52428800 0.001525 signed -xplain1.4-7base-32k-step1743 Layer 35 | input_layernorm Layer 35 input_layernorm 4096 0.001522 signed -xplain1.4-7base-32k-step1743 Layer 29 | mlp.down_proj Layer 29 mlp.down_proj 52428800 0.001521 signed -xplain1.4-7base-32k-step1743 Layer 31 | mlp.up_proj Layer 31 mlp.up_proj 52428800 0.001520 signed -xplain1.4-7base-32k-step1743 Layer 29 | mlp.gate_proj Layer 29 mlp.gate_proj 52428800 0.001518 signed -xplain1.4-7base-32k-step1743 Layer 39 | self_attn.q_proj Layer 39 self_attn.q_proj 16777216 0.001518 signed -xplain1.4-7base-32k-step1743 Layer 34 | mlp.down_proj Layer 34 mlp.down_proj 52428800 0.001517 signed -xplain1.4-7base-32k-step1743 Layer 32 | mlp.up_proj Layer 32 mlp.up_proj 52428800 0.001515 signed -xplain1.4-7base-32k-step1743 Layer 37 | self_attn.q_proj Layer 37 self_attn.q_proj 16777216 0.001511 signed + model sublayer block_label module_type count std stat_mode +xplain1.4-7base-32k-step1743 Layer 22 | input_layernorm Layer 22 input_layernorm 4096 0.004702 signed +xplain1.4-7base-32k-step1743 Layer 28 | input_layernorm Layer 28 input_layernorm 4096 0.004587 signed +xplain1.4-7base-32k-step1743 Layer 25 | input_layernorm Layer 25 input_layernorm 4096 0.004584 signed +xplain1.4-7base-32k-step1743 Layer 18 | input_layernorm Layer 18 input_layernorm 4096 0.004540 signed +xplain1.4-7base-32k-step1743 Layer 21 | input_layernorm Layer 21 input_layernorm 4096 0.002993 signed +xplain1.4-7base-32k-step1743 Layer 38 | input_layernorm Layer 38 input_layernorm 4096 0.002686 signed +xplain1.4-7base-32k-step1743 Layer 17 | input_layernorm Layer 17 input_layernorm 4096 0.002019 signed +xplain1.4-7base-32k-step1743 Layer 36 | input_layernorm Layer 36 input_layernorm 4096 0.001978 signed +xplain1.4-7base-32k-step1743 Layer 37 | input_layernorm Layer 37 input_layernorm 4096 0.001967 signed +xplain1.4-7base-32k-step1743 Layer 16 | input_layernorm Layer 16 input_layernorm 4096 0.001832 signed +xplain1.4-7base-32k-step1743 Layer 14 | input_layernorm Layer 14 input_layernorm 4096 0.001804 signed +xplain1.4-7base-32k-step1743 Layer 15 | input_layernorm Layer 15 input_layernorm 4096 0.001766 signed +xplain1.4-7base-32k-step1743 Layer 38 | mlp.gate_proj Layer 38 mlp.gate_proj 52428800 0.001752 signed +xplain1.4-7base-32k-step1743 Layer 37 | mlp.gate_proj Layer 37 mlp.gate_proj 52428800 0.001746 signed +xplain1.4-7base-32k-step1743 Layer 36 | mlp.gate_proj Layer 36 mlp.gate_proj 52428800 0.001724 signed +xplain1.4-7base-32k-step1743 Layer 37 | mlp.up_proj Layer 37 mlp.up_proj 52428800 0.001685 signed +xplain1.4-7base-32k-step1743 Layer 38 | mlp.up_proj Layer 38 mlp.up_proj 52428800 0.001680 signed +xplain1.4-7base-32k-step1743 Layer 36 | mlp.up_proj Layer 36 mlp.up_proj 52428800 0.001668 signed +xplain1.4-7base-32k-step1743 Layer 39 | post_attention_layernorm Layer 39 post_attention_layernorm 4096 0.001665 signed +xplain1.4-7base-32k-step1743 Layer 39 | mlp.gate_proj Layer 39 mlp.gate_proj 52428800 0.001665 signed +xplain1.4-7base-32k-step1743 Layer 35 | mlp.gate_proj Layer 35 mlp.gate_proj 52428800 0.001662 signed +xplain1.4-7base-32k-step1743 Layer 34 | mlp.gate_proj Layer 34 mlp.gate_proj 52428800 0.001657 signed +xplain1.4-7base-32k-step1743 Layer 19 | input_layernorm Layer 19 input_layernorm 4096 0.001644 signed +xplain1.4-7base-32k-step1743 Layer 13 | input_layernorm Layer 13 input_layernorm 4096 0.001611 signed +xplain1.4-7base-32k-step1743 Layer 00 | input_layernorm Layer 00 input_layernorm 4096 0.001606 signed +xplain1.4-7base-32k-step1743 Layer 35 | mlp.up_proj Layer 35 mlp.up_proj 52428800 0.001599 signed +xplain1.4-7base-32k-step1743 Layer 34 | mlp.up_proj Layer 34 mlp.up_proj 52428800 0.001599 signed +xplain1.4-7base-32k-step1743 Layer 33 | mlp.gate_proj Layer 33 mlp.gate_proj 52428800 0.001592 signed +xplain1.4-7base-32k-step1743 Layer 34 | self_attn.q_proj Layer 34 self_attn.q_proj 16777216 0.001577 signed +xplain1.4-7base-32k-step1743 Layer 36 | self_attn.q_proj Layer 36 self_attn.q_proj 16777216 0.001575 signed +xplain1.4-7base-32k-step1743 Layer 32 | mlp.gate_proj Layer 32 mlp.gate_proj 52428800 0.001565 signed +xplain1.4-7base-32k-step1743 Layer 31 | mlp.gate_proj Layer 31 mlp.gate_proj 52428800 0.001558 signed +xplain1.4-7base-32k-step1743 Layer 35 | self_attn.q_proj Layer 35 self_attn.q_proj 16777216 0.001557 signed +xplain1.4-7base-32k-step1743 Layer 38 | self_attn.q_proj Layer 38 self_attn.q_proj 16777216 0.001556 signed +xplain1.4-7base-32k-step1743 Layer 39 | mlp.up_proj Layer 39 mlp.up_proj 52428800 0.001542 signed +xplain1.4-7base-32k-step1743 Layer 30 | input_layernorm Layer 30 input_layernorm 4096 0.001540 signed +xplain1.4-7base-32k-step1743 Layer 30 | mlp.gate_proj Layer 30 mlp.gate_proj 52428800 0.001533 signed +xplain1.4-7base-32k-step1743 Layer 33 | mlp.up_proj Layer 33 mlp.up_proj 52428800 0.001533 signed +xplain1.4-7base-32k-step1743 Layer 31 | mlp.down_proj Layer 31 mlp.down_proj 52428800 0.001528 signed +xplain1.4-7base-32k-step1743 Layer 30 | mlp.down_proj Layer 30 mlp.down_proj 52428800 0.001525 signed +xplain1.4-7base-32k-step1743 Layer 35 | input_layernorm Layer 35 input_layernorm 4096 0.001522 signed +xplain1.4-7base-32k-step1743 Layer 29 | mlp.down_proj Layer 29 mlp.down_proj 52428800 0.001521 signed +xplain1.4-7base-32k-step1743 Layer 31 | mlp.up_proj Layer 31 mlp.up_proj 52428800 0.001520 signed +xplain1.4-7base-32k-step1743 Layer 29 | mlp.gate_proj Layer 29 mlp.gate_proj 52428800 0.001518 signed +xplain1.4-7base-32k-step1743 Layer 39 | self_attn.q_proj Layer 39 self_attn.q_proj 16777216 0.001518 signed +xplain1.4-7base-32k-step1743 Layer 34 | mlp.down_proj Layer 34 mlp.down_proj 52428800 0.001517 signed +xplain1.4-7base-32k-step1743 Layer 32 | mlp.up_proj Layer 32 mlp.up_proj 52428800 0.001515 signed +xplain1.4-7base-32k-step1743 Layer 37 | self_attn.q_proj Layer 37 self_attn.q_proj 16777216 0.001511 signed +xplain1.4-7base-32k-step1743 Layer 28 | mlp.gate_proj Layer 28 mlp.gate_proj 52428800 0.001510 signed +xplain1.4-7base-32k-step1743 Layer 36 | mlp.down_proj Layer 36 mlp.down_proj 52428800 0.001509 signed ## Top layers or global blocks by std ### `xplain1.3-6base-16k-step2199` model block_label count std stat_mode -xplain1.3-6base-16k-step2199 Final Norm 4096 0.003860 signed -xplain1.3-6base-16k-step2199 Embedding 201326592 0.001496 signed xplain1.3-6base-16k-step2199 Layer 36 199237632 0.001477 signed xplain1.3-6base-16k-step2199 Layer 37 199237632 0.001473 signed xplain1.3-6base-16k-step2199 Layer 38 199237632 0.001456 signed @@ -600,8 +598,6 @@ xplain1.3-6base-16k-step2199 Layer 02 199237632 0.000980 signed ### `xplain1.3-6base-32k-step1647` model block_label count std stat_mode -xplain1.3-6base-32k-step1647 Final Norm 4096 0.005347 signed -xplain1.3-6base-32k-step1647 Embedding 201326592 0.001762 signed xplain1.3-6base-32k-step1647 Layer 36 199237632 0.001691 signed xplain1.3-6base-32k-step1647 Layer 37 199237632 0.001681 signed xplain1.3-6base-32k-step1647 Layer 38 199237632 0.001660 signed @@ -646,8 +642,6 @@ xplain1.3-6base-32k-step1647 Layer 02 199237632 0.001088 signed ### `xplain1.3-6base-64k-step1647` model block_label count std stat_mode -xplain1.3-6base-64k-step1647 Final Norm 4096 0.006869 signed -xplain1.3-6base-64k-step1647 Embedding 201326592 0.002014 signed xplain1.3-6base-64k-step1647 Layer 36 199237632 0.001877 signed xplain1.3-6base-64k-step1647 Layer 37 199237632 0.001862 signed xplain1.3-6base-64k-step1647 Layer 34 199237632 0.001847 signed @@ -692,13 +686,11 @@ xplain1.3-6base-64k-step1647 Layer 03 199237632 0.001333 signed ### `xplain1.3-6base-8k-step2199` model block_label count std stat_mode -xplain1.3-6base-8k-step2199 Final Norm 4096 0.002388 signed xplain1.3-6base-8k-step2199 Layer 37 199237632 0.001120 signed xplain1.3-6base-8k-step2199 Layer 36 199237632 0.001117 signed xplain1.3-6base-8k-step2199 Layer 38 199237632 0.001110 signed xplain1.3-6base-8k-step2199 Layer 34 199237632 0.001072 signed xplain1.3-6base-8k-step2199 Layer 35 199237632 0.001063 signed -xplain1.3-6base-8k-step2199 Embedding 201326592 0.001027 signed xplain1.3-6base-8k-step2199 Layer 39 199237632 0.000948 signed xplain1.3-6base-8k-step2199 Layer 31 199237632 0.000937 signed xplain1.3-6base-8k-step2199 Layer 33 199237632 0.000935 signed @@ -738,10 +730,8 @@ xplain1.3-6base-8k-step2199 Layer 01 199237632 0.000535 signed ### `xplain1.4-6base-16k-step2319` model block_label count std stat_mode -xplain1.4-6base-16k-step2319 Final Norm 4096 0.005594 signed xplain1.4-6base-16k-step2319 Layer 38 199237632 0.001784 signed xplain1.4-6base-16k-step2319 Layer 37 199237632 0.001714 signed -xplain1.4-6base-16k-step2319 Embedding 201326592 0.001679 signed xplain1.4-6base-16k-step2319 Layer 39 199237632 0.001580 signed xplain1.4-6base-16k-step2319 Layer 36 199237632 0.001482 signed xplain1.4-6base-16k-step2319 Layer 35 199237632 0.001424 signed @@ -784,8 +774,6 @@ xplain1.4-6base-16k-step2319 Layer 03 199237632 0.001000 signed ### `xplain1.4-6base-32k-step1743` model block_label count std stat_mode -xplain1.4-6base-32k-step1743 Final Norm 4096 0.006660 signed -xplain1.4-6base-32k-step1743 Embedding 201326592 0.002007 signed xplain1.4-6base-32k-step1743 Layer 38 199237632 0.001992 signed xplain1.4-6base-32k-step1743 Layer 37 199237632 0.001929 signed xplain1.4-6base-32k-step1743 Layer 39 199237632 0.001778 signed @@ -830,8 +818,6 @@ xplain1.4-6base-32k-step1743 Layer 07 199237632 0.001267 signed ### `xplain1.4-6base-64k-step1743` model block_label count std stat_mode -xplain1.4-6base-64k-step1743 Final Norm 4096 0.007889 signed -xplain1.4-6base-64k-step1743 Embedding 201326592 0.002247 signed xplain1.4-6base-64k-step1743 Layer 38 199237632 0.002120 signed xplain1.4-6base-64k-step1743 Layer 37 199237632 0.002040 signed xplain1.4-6base-64k-step1743 Layer 39 199237632 0.001909 signed @@ -876,11 +862,9 @@ xplain1.4-6base-64k-step1743 Layer 03 199237632 0.001353 signed ### `xplain1.4-6base-8k-step2319` model block_label count std stat_mode -xplain1.4-6base-8k-step2319 Final Norm 4096 0.003269 signed xplain1.4-6base-8k-step2319 Layer 38 199237632 0.001407 signed xplain1.4-6base-8k-step2319 Layer 37 199237632 0.001340 signed xplain1.4-6base-8k-step2319 Layer 39 199237632 0.001194 signed -xplain1.4-6base-8k-step2319 Embedding 201326592 0.001185 signed xplain1.4-6base-8k-step2319 Layer 36 199237632 0.001110 signed xplain1.4-6base-8k-step2319 Layer 35 199237632 0.001067 signed xplain1.4-6base-8k-step2319 Layer 34 199237632 0.001059 signed @@ -922,8 +906,6 @@ xplain1.4-6base-8k-step2319 Layer 01 199237632 0.000724 signed ### `xplain1.4-7base-32k-step1743` model block_label count std stat_mode -xplain1.4-7base-32k-step1743 Final Norm 4096 0.006105 signed -xplain1.4-7base-32k-step1743 Embedding 201326592 0.001716 signed xplain1.4-7base-32k-step1743 Layer 36 199237632 0.001582 signed xplain1.4-7base-32k-step1743 Layer 38 199237632 0.001579 signed xplain1.4-7base-32k-step1743 Layer 37 199237632 0.001578 signed @@ -970,9 +952,7 @@ xplain1.4-7base-32k-step1743 Layer 00 199237632 0.000917 signed ### `xplain1.3-6base-16k-step2199` model module_type count std stat_mode -xplain1.3-6base-16k-step2199 final_norm 4096 0.003860 signed xplain1.3-6base-16k-step2199 input_layernorm 163840 0.002186 signed -xplain1.3-6base-16k-step2199 embed_tokens 201326592 0.001496 signed xplain1.3-6base-16k-step2199 mlp.gate_proj 2097152000 0.001318 signed xplain1.3-6base-16k-step2199 post_attention_layernorm 163840 0.001232 signed xplain1.3-6base-16k-step2199 mlp.up_proj 2097152000 0.001229 signed @@ -985,9 +965,7 @@ xplain1.3-6base-16k-step2199 self_attn.v_proj 167772160 0.000902 sig ### `xplain1.3-6base-32k-step1647` model module_type count std stat_mode -xplain1.3-6base-32k-step1647 final_norm 4096 0.005347 signed xplain1.3-6base-32k-step1647 input_layernorm 163840 0.002933 signed -xplain1.3-6base-32k-step1647 embed_tokens 201326592 0.001762 signed xplain1.3-6base-32k-step1647 mlp.gate_proj 2097152000 0.001524 signed xplain1.3-6base-32k-step1647 post_attention_layernorm 163840 0.001465 signed xplain1.3-6base-32k-step1647 mlp.up_proj 2097152000 0.001415 signed @@ -1000,9 +978,7 @@ xplain1.3-6base-32k-step1647 self_attn.v_proj 167772160 0.001018 sig ### `xplain1.3-6base-64k-step1647` model module_type count std stat_mode -xplain1.3-6base-64k-step1647 final_norm 4096 0.006869 signed xplain1.3-6base-64k-step1647 input_layernorm 163840 0.003692 signed -xplain1.3-6base-64k-step1647 embed_tokens 201326592 0.002014 signed xplain1.3-6base-64k-step1647 post_attention_layernorm 163840 0.001730 signed xplain1.3-6base-64k-step1647 mlp.gate_proj 2097152000 0.001716 signed xplain1.3-6base-64k-step1647 mlp.up_proj 2097152000 0.001599 signed @@ -1015,9 +991,7 @@ xplain1.3-6base-64k-step1647 self_attn.v_proj 167772160 0.001183 sig ### `xplain1.3-6base-8k-step2199` model module_type count std stat_mode -xplain1.3-6base-8k-step2199 final_norm 4096 0.002388 signed xplain1.3-6base-8k-step2199 input_layernorm 163840 0.001203 signed -xplain1.3-6base-8k-step2199 embed_tokens 201326592 0.001027 signed xplain1.3-6base-8k-step2199 mlp.gate_proj 2097152000 0.000944 signed xplain1.3-6base-8k-step2199 mlp.up_proj 2097152000 0.000870 signed xplain1.3-6base-8k-step2199 mlp.down_proj 2097152000 0.000842 signed @@ -1030,9 +1004,7 @@ xplain1.3-6base-8k-step2199 self_attn.v_proj 167772160 0.000571 sign ### `xplain1.4-6base-16k-step2319` model module_type count std stat_mode -xplain1.4-6base-16k-step2319 final_norm 4096 0.005594 signed xplain1.4-6base-16k-step2319 input_layernorm 163840 0.002340 signed -xplain1.4-6base-16k-step2319 embed_tokens 201326592 0.001679 signed xplain1.4-6base-16k-step2319 mlp.gate_proj 2097152000 0.001364 signed xplain1.4-6base-16k-step2319 post_attention_layernorm 163840 0.001322 signed xplain1.4-6base-16k-step2319 mlp.up_proj 2097152000 0.001263 signed @@ -1045,9 +1017,7 @@ xplain1.4-6base-16k-step2319 self_attn.v_proj 167772160 0.000919 sig ### `xplain1.4-6base-32k-step1743` model module_type count std stat_mode -xplain1.4-6base-32k-step1743 final_norm 4096 0.006660 signed xplain1.4-6base-32k-step1743 input_layernorm 163840 0.003153 signed -xplain1.4-6base-32k-step1743 embed_tokens 201326592 0.002007 signed xplain1.4-6base-32k-step1743 mlp.gate_proj 2097152000 0.001612 signed xplain1.4-6base-32k-step1743 post_attention_layernorm 163840 0.001604 signed xplain1.4-6base-32k-step1743 mlp.up_proj 2097152000 0.001500 signed @@ -1060,9 +1030,7 @@ xplain1.4-6base-32k-step1743 self_attn.v_proj 167772160 0.001136 sig ### `xplain1.4-6base-64k-step1743` model module_type count std stat_mode -xplain1.4-6base-64k-step1743 final_norm 4096 0.007889 signed xplain1.4-6base-64k-step1743 input_layernorm 163840 0.003853 signed -xplain1.4-6base-64k-step1743 embed_tokens 201326592 0.002247 signed xplain1.4-6base-64k-step1743 post_attention_layernorm 163840 0.001784 signed xplain1.4-6base-64k-step1743 mlp.gate_proj 2097152000 0.001755 signed xplain1.4-6base-64k-step1743 mlp.up_proj 2097152000 0.001630 signed @@ -1075,9 +1043,7 @@ xplain1.4-6base-64k-step1743 self_attn.v_proj 167772160 0.001245 sig ### `xplain1.4-6base-8k-step2319` model module_type count std stat_mode -xplain1.4-6base-8k-step2319 final_norm 4096 0.003269 signed xplain1.4-6base-8k-step2319 input_layernorm 163840 0.001287 signed -xplain1.4-6base-8k-step2319 embed_tokens 201326592 0.001185 signed xplain1.4-6base-8k-step2319 mlp.gate_proj 2097152000 0.001025 signed xplain1.4-6base-8k-step2319 mlp.up_proj 2097152000 0.000950 signed xplain1.4-6base-8k-step2319 mlp.down_proj 2097152000 0.000910 signed @@ -1090,9 +1056,7 @@ xplain1.4-6base-8k-step2319 self_attn.v_proj 167772160 0.000709 sign ### `xplain1.4-7base-32k-step1743` model module_type count std stat_mode -xplain1.4-7base-32k-step1743 final_norm 4096 0.006105 signed xplain1.4-7base-32k-step1743 input_layernorm 163840 0.002731 signed -xplain1.4-7base-32k-step1743 embed_tokens 201326592 0.001716 signed xplain1.4-7base-32k-step1743 post_attention_layernorm 163840 0.001456 signed xplain1.4-7base-32k-step1743 mlp.gate_proj 2097152000 0.001452 signed xplain1.4-7base-32k-step1743 mlp.up_proj 2097152000 0.001344 signed